From ff490bed5ba5694cde305d3bc63f30a8f1b3f1e2 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 16 Aug 2022 02:04:03 +0300 Subject: [PATCH 001/420] Small fixes. --- cmd/harmony/main.go | 9 ++++----- consensus/view_change_construct.go | 2 +- rpc/blockchain.go | 3 +-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index da51d9c52e..437a91dc59 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -769,7 +769,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi ) nodeconfig.GetDefaultConfig().DBDir = nodeConfig.DBDir - processNodeType(hc, currentNode, currentConsensus) + currentConsensus.SetIsBackup(processNodeType(hc, currentNode)) currentNode.NodeConfig.SetShardGroupID(nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(nodeConfig.ShardID))) currentNode.NodeConfig.SetClientGroupID(nodeconfig.NewClientGroupIDByShardID(shard.BeaconChainShardID)) currentNode.NodeConfig.ConsensusPriKey = nodeConfig.ConsensusPriKey @@ -821,7 +821,7 @@ func setupTiKV(hc harmonyconfig.HarmonyConfig) shardchain.DBFactory { return factory } -func processNodeType(hc harmonyconfig.HarmonyConfig, currentNode *node.Node, currentConsensus *consensus.Consensus) { +func processNodeType(hc harmonyconfig.HarmonyConfig, currentNode *node.Node) (isBackup bool) { switch hc.General.NodeType { case nodeTypeExplorer: nodeconfig.SetDefaultRole(nodeconfig.ExplorerNode) @@ -831,10 +831,9 @@ func processNodeType(hc harmonyconfig.HarmonyConfig, currentNode *node.Node, cur nodeconfig.SetDefaultRole(nodeconfig.Validator) currentNode.NodeConfig.SetRole(nodeconfig.Validator) - if hc.General.IsBackup { - currentConsensus.SetIsBackup(true) - } + return hc.General.IsBackup } + return false } func setupPprofService(node *node.Node, hc harmonyconfig.HarmonyConfig) { diff --git a/consensus/view_change_construct.go b/consensus/view_change_construct.go index b818507eb7..52ea8bbd72 100644 --- a/consensus/view_change_construct.go +++ b/consensus/view_change_construct.go @@ -87,7 +87,7 @@ func (vc *viewChange) AddViewIDKeyIfNotExist(viewID uint64, members multibls.Pub } } -// Reset reset the state for viewchange +// Reset resets the state for viewChange. func (vc *viewChange) Reset() { vc.m1Payload = []byte{} vc.bhpSigs = map[uint64]map[string]*bls_core.Sign{} diff --git a/rpc/blockchain.go b/rpc/blockchain.go index 6f3e2293f7..46e495944d 100644 --- a/rpc/blockchain.go +++ b/rpc/blockchain.go @@ -2,13 +2,12 @@ package rpc import ( "context" + "encoding/hex" "fmt" "math/big" "reflect" "time" - "encoding/hex" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" From 85dd4acfbaf62833575e10788e2b3a06f6bac26e Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 1 Sep 2022 00:53:16 +0700 Subject: [PATCH 002/420] in progress. --- api/service/blockproposal/service.go | 15 +++++---- api/service/explorer/service.go | 48 ++++++++++++++++++++++++++++ cmd/harmony/main.go | 6 +++- consensus/checks.go | 3 +- consensus/consensus.go | 6 +++- consensus/consensus_service.go | 48 ++++++++++++++++++++++++---- consensus/consensus_v2.go | 33 +++++++++++++++++-- consensus/leader.go | 12 +++++-- consensus/quorum/quorum.go | 7 +++- consensus/validator.go | 14 ++++++++ consensus/view_change.go | 30 +++++++++++++++-- hmy/hmy.go | 2 ++ internal/utils/singleton.go | 16 ++++++++++ node/api.go | 6 ++++ node/node_newblock.go | 4 +++ test/configs/local-resharding.txt | 31 ++++++------------ test/deploy.sh | 5 ++- 17 files changed, 237 insertions(+), 49 deletions(-) diff --git a/api/service/blockproposal/service.go b/api/service/blockproposal/service.go index 4d42c558aa..1cbb5accf3 100644 --- a/api/service/blockproposal/service.go +++ b/api/service/blockproposal/service.go @@ -18,19 +18,22 @@ type Service struct { // New returns a block proposal service. func New(readySignal chan consensus.ProposalType, commitSigsChan chan []byte, waitForConsensusReady func(readySignal chan consensus.ProposalType, commitSigsChan chan []byte, stopChan chan struct{}, stoppedChan chan struct{})) *Service { - return &Service{readySignal: readySignal, commitSigsChan: commitSigsChan, waitForConsensusReady: waitForConsensusReady} + return &Service{ + readySignal: readySignal, + commitSigsChan: commitSigsChan, + waitForConsensusReady: waitForConsensusReady, + stopChan: make(chan struct{}), + stoppedChan: make(chan struct{}), + } } // Start starts block proposal service. func (s *Service) Start() error { - s.stopChan = make(chan struct{}) - s.stoppedChan = make(chan struct{}) - - s.run(s.stopChan, s.stoppedChan) + s.run() return nil } -func (s *Service) run(stopChan chan struct{}, stoppedChan chan struct{}) { +func (s *Service) run() { s.waitForConsensusReady(s.readySignal, s.commitSigsChan, s.stopChan, s.stoppedChan) } diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index b5ffa1b8b0..d96c1bc56c 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -7,11 +7,13 @@ import ( "fmt" "net" "net/http" + "os" "path" "strconv" "time" "github.com/RoaringBitmap/roaring/roaring64" + "github.com/harmony-one/harmony/internal/common" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" ethCommon "github.com/ethereum/go-ethereum/common" @@ -113,12 +115,17 @@ func (s *Service) Run() *http.Server { s.router = mux.NewRouter() + fmt.Println("++", addr) + // Set up router for addresses. // Fetch addresses request, accepts parameter size: how much addresses to read, // parameter prefix: from which address prefix start s.router.Path("/addresses").Queries("size", "{[0-9]*?}", "prefix", "{[a-zA-Z0-9]*?}").HandlerFunc(s.GetAddresses).Methods("GET") s.router.Path("/addresses").HandlerFunc(s.GetAddresses) s.router.Path("/height").HandlerFunc(s.GetHeight) + s.router.Path("/leader").HandlerFunc(s.GetLeader) + s.router.Path("/blocks").HandlerFunc(s.GetBlocks) + s.router.Path("/halt").HandlerFunc(s.halt) // Set up router for supply info s.router.Path("/burn-addresses").Queries().HandlerFunc(s.GetInaccessibleAddressInfo).Methods("GET") @@ -186,6 +193,47 @@ type HeightResponse struct { S3 uint64 `json:"3,omitempty"` } +func (s *Service) GetLeader(w http.ResponseWriter, r *http.Request) { + if s.backend.IsCurrentlyLeader() { + w.Write([]byte("true ")) + } else { + w.Write([]byte("false")) + } + + keys := "" + for _, p := range s.backend.GetPublicKeys() { + addr := common.Address{} + addrBytes := p.Object.GetAddress() + addr.SetBytes(addrBytes[:]) + keys += fmt.Sprintf("%s ", addr.String()) + break + } + //blsPubKeyBytes := leaderKey.Object.GetAddress() + //coinbase.SetBytes(blsPubKeyBytes[:]) + + w.Write([]byte(fmt.Sprintf(" %d", s.blockchain.ShardID()))) + w.Write([]byte(fmt.Sprintf(" %s", s.Port))) + w.Write([]byte(fmt.Sprintf(" %s", keys))) + w.Write([]byte(fmt.Sprintf(" %s", s.backend.GetPublicKeys().SerializeToHexStr()))) + +} + +func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { + cur := s.blockchain.CurrentHeader().Number().Uint64() + + for i := cur; i > 0; i-- { + block := s.blockchain.GetBlockByNumber(i) + w.Write([]byte(fmt.Sprintf("%d ", i))) + w.Write([]byte(fmt.Sprintf("%s ", block.Header().ViewID().String()))) + w.Write([]byte(fmt.Sprintf("%s ", block.Header().Coinbase().Hash().Hex()))) + w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) + } +} + +func (s *Service) halt(w http.ResponseWriter, r *http.Request) { + os.Exit(0) +} + // GetHeight returns heights of current and beacon chains if needed. func (s *Service) GetHeight(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 437a91dc59..75eabfb2e8 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -265,6 +265,8 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { var err error + fmt.Println("OS: ", os.Args) + nodeconfigSetShardSchedule(hc) nodeconfig.SetShardingSchedule(shard.Schedule) nodeconfig.SetVersion(getHarmonyVersion()) @@ -420,8 +422,9 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { if currentNode.NodeConfig.Role() == nodeconfig.Validator { currentNode.RegisterValidatorServices() } else if currentNode.NodeConfig.Role() == nodeconfig.ExplorerNode { - currentNode.RegisterExplorerServices() + } + currentNode.RegisterExplorerServices() currentNode.RegisterService(service.CrosslinkSending, crosslink_sending.New(currentNode, currentNode.Blockchain())) if hc.Pprof.Enabled { setupPprofService(currentNode, hc) @@ -784,6 +787,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // Set the consensus ID to be the current block number viewID := currentNode.Blockchain().CurrentBlock().Header().ViewID().Uint64() + fmt.Println("viewID:", viewID) currentConsensus.SetViewIDs(viewID + 1) utils.Logger().Info(). Uint64("viewID", viewID). diff --git a/consensus/checks.go b/consensus/checks.go index ceaf9987b9..c44a58da39 100644 --- a/consensus/checks.go +++ b/consensus/checks.go @@ -55,8 +55,7 @@ func (consensus *Consensus) senderKeySanityChecks(msg *msg_pb.Message, senderKey return true } -func (consensus *Consensus) isRightBlockNumAndViewID(recvMsg *FBFTMessage, -) bool { +func (consensus *Consensus) isRightBlockNumAndViewID(recvMsg *FBFTMessage) bool { if recvMsg.ViewID != consensus.GetCurBlockViewID() || recvMsg.BlockNum != consensus.BlockNum() { consensus.getLogger().Debug(). Uint64("blockNum", consensus.BlockNum()). diff --git a/consensus/consensus.go b/consensus/consensus.go index 89897e372d..2598010102 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -73,6 +73,8 @@ type Consensus struct { priKey multibls.PrivateKeys // the publickey of leader LeaderPubKey *bls.PublicKeyWrapper + // index of leader in the list of validators. + LeaderIndex int // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 @@ -220,7 +222,9 @@ func New( registry *registry.Registry, Decider quorum.Decider, minPeers int, aggregateSig bool, ) (*Consensus, error) { - consensus := Consensus{} + consensus := Consensus{ + ShardID: shard, + } consensus.Decider = Decider consensus.registry = registry consensus.MinPeers = minPeers diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 310c9bb9da..e646504dde 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync/atomic" "time" @@ -106,11 +107,6 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi return consensus.Decider.ParticipantsCount() } -// NewFaker returns a faker consensus. -func NewFaker() *Consensus { - return &Consensus{} -} - // Sign on the hash of the message func (consensus *Consensus) signMessage(message []byte, priKey *bls_core.SecretKey) []byte { hash := hash.Keccak256(message) @@ -217,6 +213,7 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } + fmt.Println("[checkViewID] Set LEADEER PUB KEY ", msg.SenderPubkeys[0].Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() @@ -400,6 +397,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { Str("leaderPubKey", leaderPubKey.Bytes.Hex()). Msg("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain") consensus.pubKeyLock.Lock() + fmt.Println("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain", leaderPubKey.Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = leaderPubKey consensus.pubKeyLock.Unlock() } @@ -451,16 +449,43 @@ func (consensus *Consensus) IsLeader() bool { return false } +// isLeader check if the node is a leader or not by comparing the public key of +// the node with the leader public key. This function assume it runs under lock. +func (consensus *Consensus) isLeader() bool { + obj := consensus.LeaderPubKey.Object + for _, key := range consensus.priKey { + if key.Pub.Object.IsEqual(obj) { + return true + } + } + return false +} + // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { + fmt.Println("SetViewIDs", height) consensus.SetCurBlockViewID(height) consensus.SetViewChangingID(height) } // SetCurBlockViewID set the current view ID -func (consensus *Consensus) SetCurBlockViewID(viewID uint64) { - consensus.current.SetCurBlockViewID(viewID) +func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { + return consensus.current.SetCurBlockViewID(viewID) +} + +// SetLeaderIndex set the leader index. +func (consensus *Consensus) SetLeaderIndex(f func(int) int) (current int) { + consensus.pubKeyLock.Lock() + defer consensus.pubKeyLock.Unlock() + consensus.LeaderIndex = f(consensus.LeaderIndex) + return consensus.LeaderIndex +} + +func (consensus *Consensus) GetLeaderIndex() int { + consensus.pubKeyLock.Lock() + defer consensus.pubKeyLock.Unlock() + return consensus.LeaderIndex } // SetViewChangingID set the current view change ID @@ -473,6 +498,15 @@ func (consensus *Consensus) StartFinalityCount() { consensus.finalityCounter.Store(time.Now().UnixNano()) } +//func (consensus *Consensus) ReshardingNextLeader(newblock *types.Block) { +// consensus.pubKeyLock.Lock() +// fmt.Println("nextBlock1 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) +// consensus.LeaderPubKey = consensus.getNextLeaderKey(consensus.GetCurBlockViewID() + 1) +// fmt.Println("nextBlock2 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) +// consensus.pubKeyLock.Unlock() +// +//} + // FinishFinalityCount calculate the current finality func (consensus *Consensus) FinishFinalityCount() { d := time.Now().UnixNano() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index deb0883d94..16554d0393 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/hex" + "fmt" "sync/atomic" "time" @@ -130,6 +131,7 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb } func (consensus *Consensus) finalCommit() { + // THIS IS NOT GOOD PLACE FOR LEADER SWITCHING numCommits := consensus.Decider.SignersCount(quorum.Commit) consensus.getLogger().Info(). @@ -392,6 +394,7 @@ func (consensus *Consensus) Start( consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() case newBlock := <-blockChannel: + //consensus.ReshardingNextLeader(newBlock) consensus.getLogger().Info(). Uint64("MsgBlockNum", newBlock.NumberU64()). Msg("[ConsensusMainLoop] Received Proposed New Block!") @@ -403,6 +406,7 @@ func (consensus *Consensus) Start( } // Sleep to wait for the full block time consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") + <-time.After(time.Until(consensus.NextBlockDue)) consensus.StartFinalityCount() @@ -431,7 +435,7 @@ func (consensus *Consensus) Start( } } -// Close close the consensus. If current is in normal commit phase, wait until the commit +// Close closes the consensus. If current is in normal commit phase, wait until the commit // phase end. func (consensus *Consensus) Close() error { if consensus.dHelper != nil { @@ -527,6 +531,7 @@ func (consensus *Consensus) getLastMileBlocksAndMsg(bnStart uint64) ([]*types.Bl // preCommitAndPropose commit the current block with 67% commit signatures and start // proposing new block which will wait on the full commit signatures to finish func (consensus *Consensus) preCommitAndPropose(blk *types.Block) error { + //fmt.Println("preCommitAndPropose", utils.GetPort(), blk.NumberU64()) if blk == nil { return errors.New("block to pre-commit is nil") } @@ -644,6 +649,7 @@ func (consensus *Consensus) tryCatchup() error { consensus.getLogger().Error().Err(err).Msg("[TryCatchup] Failed to add block to chain") return err } + //fmt.Println("tryCatchup ", utils.GetPort(), blk.NumberU64()) select { // TODO: Remove this when removing dns sync and stream sync is fully up case consensus.VerifiedNewBlock <- blk: @@ -658,6 +664,8 @@ func (consensus *Consensus) tryCatchup() error { } func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMessage) error { + // this function evaluates for all, leader and validators. + if consensus.Blockchain().CurrentBlock().NumberU64() < blk.NumberU64() { if _, err := consensus.Blockchain().InsertChain([]*types.Block{blk}, !consensus.FBFTLog.IsBlockVerified(blk.Hash())); err != nil { consensus.getLogger().Error().Err(err).Msg("[commitBlock] Failed to add block to chain") @@ -682,10 +690,29 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess // SetupForNewConsensus sets the state for new consensus func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) - consensus.SetCurBlockViewID(committedMsg.ViewID + 1) + curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) // first view id is going to be 2. + prev := consensus.GetLeaderPubKey() + idx := consensus.SetLeaderIndex(func(i int) int { + if curBlockViewID%3 == 0 { + return i + 1 + } + return i + }) + pps := consensus.Decider.Participants() consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] + consensus.LeaderPubKey = &pps[idx%len(pps)] + fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %q\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) consensus.pubKeyLock.Unlock() + if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { + // leader changed + go func() { + fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + consensus.ReadySignal <- SyncProposal + + }() + + } // Update consensus keys at last so the change of leader status doesn't mess up normal flow if blk.IsLastBlockInEpoch() { consensus.SetMode(consensus.UpdateConsensusInformation()) diff --git a/consensus/leader.go b/consensus/leader.go index 477d8eb299..a359c229a4 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -3,12 +3,11 @@ package consensus import ( "time" + "github.com/harmony-one/harmony/consensus/signature" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/common" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" - "github.com/harmony-one/harmony/consensus/signature" - "github.com/ethereum/go-ethereum/rlp" bls_core "github.com/harmony-one/bls/ffi/go/bls" msg_pb "github.com/harmony-one/harmony/api/proto/message" @@ -200,9 +199,17 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { + // TODO HERE + //if recvMsg.ViewID == 10 { + // return + //} consensus.mutex.Lock() defer consensus.mutex.Unlock() //// Read - Start + if consensus.ShardID == 0 { + //fmt.Println("onCommit ", recvMsg.BlockNum) + } + if !consensus.isRightBlockNumAndViewID(recvMsg) { return } @@ -334,4 +341,5 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED) } + //fmt.Println("onCommit99: ", utils.GetPort(), recvMsg.BlockNum) } diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 867fd99677..7e76dfcc6b 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -231,12 +231,17 @@ func (s *cIdentities) NthNextHmy(instance shardingconfig.Instance, pubKey *bls.P Msg("[NthNextHmy] pubKey not found") } numNodes := instance.NumHarmonyOperatedNodesPerShard() + //fmt.Println("??idx:", idx, numNodes) // sanity check to avoid out of bound access if numNodes <= 0 || numNodes > len(s.publicKeys) { numNodes = len(s.publicKeys) } idx = (idx + next) % numNodes - return found, &s.publicKeys[idx] + //fmt.Println("-------idx:", idx) + + new := &s.publicKeys[idx] + fmt.Println("NthNextHmy: ", pubKey.Bytes.Hex(), new.Bytes.Hex()) + return found, new } // NthNextHmyExt return the Nth next pubkey of Harmony + allowlist nodes, next can be negative number diff --git a/consensus/validator.go b/consensus/validator.go index a73ac92eb3..013769395c 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -18,6 +18,7 @@ import ( ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { + recvMsg, err := consensus.ParseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). @@ -26,6 +27,9 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Msg("[OnAnnounce] Unparseable leader message") return } + if consensus.ShardID == 0 { + //fmt.Println("onAnnounce called ", recvMsg.BlockNum) + } // NOTE let it handle its own logs if !consensus.onAnnounceSanityChecks(recvMsg) { @@ -184,6 +188,9 @@ func (consensus *Consensus) sendCommitMessages(blockObj *types.Block) { // if onPrepared accepts the prepared message from the leader, then // it will send a COMMIT message for the leader to receive on the network. func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { + if consensus.ShardID == 0 { + //fmt.Println("onPrepared", recvMsg.BlockNum) + } consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -399,6 +406,13 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { consensus.getLogger().Info().Msg("[OnCommitted] Start consensus timer (new block added)") consensus.consensusTimeout[timeoutConsensus].Start() } + + //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum) + if blk != nil { + //consensus.ReshardingNextLeader(blk) + } else { + //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum, "blk is nil") + } } // Collect private keys that are part of the current committee. diff --git a/consensus/view_change.go b/consensus/view_change.go index 5bfd49f83d..2936fba018 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync" "time" @@ -67,10 +68,11 @@ func (pm *State) GetCurBlockViewID() uint64 { } // SetCurBlockViewID sets the current view id -func (pm *State) SetCurBlockViewID(viewID uint64) { +func (pm *State) SetCurBlockViewID(viewID uint64) uint64 { pm.cViewMux.Lock() defer pm.cViewMux.Unlock() pm.blockViewID = viewID + return pm.blockViewID } // GetViewChangingID return the current view changing id @@ -160,6 +162,7 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { Uint64("stuckBlockViewID", stuckBlockViewID). Msg("[getNextViewID]") + fmt.Println("end getNextViewID: ", nextViewID, viewChangeDuration) // duration is always the fixed view change duration for synchronous view change return nextViewID, viewChangeDuration } @@ -171,7 +174,8 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrapper { gap := 1 - if viewID > consensus.GetCurBlockViewID() { + cur := consensus.GetCurBlockViewID() + if viewID > cur { gap = int(viewID - consensus.GetCurBlockViewID()) } var lastLeaderPubKey *bls.PublicKeyWrapper @@ -231,6 +235,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe lastLeaderPubKey, gap) } + fmt.Println("wasfoundNext", consensus.Blockchain.Config().IsAllowlistEpoch(epoch), wasFound, next.Bytes.Hex(), lastLeaderPubKey.Bytes.Hex()) if !wasFound { consensus.getLogger().Warn(). Str("key", consensus.LeaderPubKey.Bytes.Hex()). @@ -252,6 +257,7 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { + fmt.Printf("Message to send leader111: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } @@ -262,6 +268,7 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() + //fmt.Println("startViewChange", nextViewID) consensus.SetViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet @@ -270,7 +277,10 @@ func (consensus *Consensus) startViewChange() { // Ideally, we shall use another variable to keep track of the // leader pubkey in viewchange mode consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = consensus.getNextLeaderKey(nextViewID) + lpk := consensus.getNextLeaderKey(nextViewID) + consensus.LeaderPubKey = lpk + //fmt.Println("Message to send leader cur: ", consensus.LeaderPubKey.Bytes.Hex(), "next: ", lpk.Bytes.Hex()) + //fmt.Println("Message to send leader: ", consensus.LeaderPubKey.Bytes.Hex()) consensus.pubKeyLock.Unlock() consensus.getLogger().Warn(). @@ -304,7 +314,9 @@ func (consensus *Consensus) startViewChange() { if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { continue } + // Тут уже другой leader msgToSend := consensus.constructViewChangeMessage(&key) + fmt.Println("Message to send leader222: ", consensus.LeaderPubKey.Bytes.Hex()) if err := consensus.msgSender.SendWithRetry( consensus.BlockNum(), msg_pb.MessageType_VIEWCHANGE, @@ -362,6 +374,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.ResetState() } + fmt.Println("[startNewView]", newLeaderPriKey.Pub.Bytes.Hex()) consensus.LeaderPubKey = newLeaderPriKey.Pub return nil @@ -369,6 +382,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri // onViewChange is called when the view change message is received. func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { + //fmt.Printf("[onViewChange] received view change message from %+v\n", recvMsg) consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -391,6 +405,13 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { return } + consensus.getLogger().Debug(). + Err(err). + Interface("SenderPubkeys", recvMsg.SenderPubkeys). + Str("NextLeader", recvMsg.LeaderPubkey.Bytes.Hex()). + Str("myBLSPubKey", consensus.priKey.GetPublicKeys().SerializeToHexStr()). + Msg("[onViewChange] I am the Leader") + if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) { consensus.getLogger().Info(). Int64("have", consensus.Decider.SignersCount(quorum.ViewChange)). @@ -471,6 +492,8 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.mutex.Lock() defer consensus.mutex.Unlock() + fmt.Printf("[onNewView] received new view message from %+v\n", recvMsg) + consensus.getLogger().Info(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -559,6 +582,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // newView message verified success, override my state consensus.SetViewIDs(recvMsg.ViewID) consensus.pubKeyLock.Lock() + fmt.Println("[onNewView1221] new leader key cur:", consensus.LeaderPubKey.Bytes.Hex(), " new: ", senderKey.Bytes.Hex()) consensus.LeaderPubKey = senderKey consensus.pubKeyLock.Unlock() consensus.ResetViewChangeState() diff --git a/hmy/hmy.go b/hmy/hmy.go index 58e8d59edf..8089f04ba8 100644 --- a/hmy/hmy.go +++ b/hmy/hmy.go @@ -18,6 +18,7 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + "github.com/harmony-one/harmony/multibls" commonRPC "github.com/harmony-one/harmony/rpc/common" "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" @@ -96,6 +97,7 @@ type NodeAPI interface { GetStakingTransactionsCount(address, txType string) (uint64, error) GetTraceResultByHash(hash common.Hash) (json.RawMessage, error) IsCurrentlyLeader() bool + GetPublicKeys() multibls.PublicKeys IsOutOfSync(shardID uint32) bool SyncStatus(shardID uint32) (bool, uint64, uint64) SyncPeers() map[string]int diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index 6409ea71e2..7aee4d86be 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path" + "strconv" "sync" "time" @@ -195,3 +196,18 @@ func updateZeroLogLevel(level int) { childLogger := Logger().Level(zeroLoggerLevel) zeroLogger = &childLogger } + +func GetPort() int { + ok := false + for _, x := range os.Args { + if x == "--port" { + ok = true + continue + } + if ok { + rs, _ := strconv.ParseInt(x, 10, 64) + return int(rs) + } + } + return 0 +} diff --git a/node/api.go b/node/api.go index ceda968084..debcb201f6 100644 --- a/node/api.go +++ b/node/api.go @@ -6,6 +6,7 @@ import ( "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy" "github.com/harmony-one/harmony/internal/tikv" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/rosetta" hmy_rpc "github.com/harmony-one/harmony/rpc" rpc_common "github.com/harmony-one/harmony/rpc/common" @@ -18,6 +19,11 @@ func (node *Node) IsCurrentlyLeader() bool { return node.Consensus.IsLeader() } +// GetPublicKeys exposes if node is currently the leader node +func (node *Node) GetPublicKeys() multibls.PublicKeys { + return node.Consensus.GetPrivateKeys().GetPublicKeys() +} + // PeerConnectivity .. func (node *Node) PeerConnectivity() (int, int, int) { return node.host.PeerConnectivity() diff --git a/node/node_newblock.go b/node/node_newblock.go index 03fd69d9dd..24e26ad99a 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -2,6 +2,7 @@ package node import ( "errors" + "fmt" "sort" "strings" "time" @@ -88,6 +89,8 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp newBlock, err := node.ProposeNewBlock(newCommitSigsChan) if err == nil { + fmt.Printf("ProposeNewBlock: #%d :%d @%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), newBlock.Header().ViewID().Int64(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) + if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). Msg("Block with the same number was already proposed, abort.") @@ -145,6 +148,7 @@ func (node *Node) ProposeNewBlock(commitSigs chan []byte) (*types.Block, error) if node.Blockchain().Config().IsStaking(header.Epoch()) { blsPubKeyBytes := leaderKey.Object.GetAddress() coinbase.SetBytes(blsPubKeyBytes[:]) + fmt.Println("coinbase.SetBytes leader: ", leaderKey.Bytes.Hex(), coinbase.Hex()) } emptyAddr := common.Address{} diff --git a/test/configs/local-resharding.txt b/test/configs/local-resharding.txt index 4ce91ece1c..fc3d3e97b4 100644 --- a/test/configs/local-resharding.txt +++ b/test/configs/local-resharding.txt @@ -1,25 +1,12 @@ 127.0.0.1 9000 validator .hmy/65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204.key -127.0.0.1 9002 validator .hmy/40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809.key -127.0.0.1 9004 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key -127.0.0.1 9006 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key -127.0.0.1 9008 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key -127.0.0.1 9010 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key -127.0.0.1 9012 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key -127.0.0.1 9014 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key -127.0.0.1 9016 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key -127.0.0.1 9018 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key -127.0.0.1 9020 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key -127.0.0.1 9022 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key -127.0.0.1 9200 explorer null 0 +127.0.0.1 9002 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key +127.0.0.1 9004 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key +127.0.0.1 9006 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key +127.0.0.1 9008 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key +127.0.0.1 9010 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key +127.0.0.1 9099 explorer null 0 127.0.0.1 9100 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key -127.0.0.1 9102 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key -127.0.0.1 9104 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key -127.0.0.1 9106 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key -127.0.0.1 9108 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key -127.0.0.1 9110 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key -127.0.0.1 9112 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key -127.0.0.1 9114 validator .hmy/f47238daef97d60deedbde5302d05dea5de67608f11f406576e363661f7dcbc4a1385948549b31a6c70f6fde8a391486.key -127.0.0.1 9116 validator .hmy/fc4b9c535ee91f015efff3f32fbb9d32cdd9bfc8a837bb3eee89b8fff653c7af2050a4e147ebe5c7233dc2d5df06ee0a.key -127.0.0.1 9118 validator .hmy/ca86e551ee42adaaa6477322d7db869d3e203c00d7b86c82ebee629ad79cb6d57b8f3db28336778ec2180e56a8e07296.key -127.0.0.1 9300 explorer null 1 \ No newline at end of file +127.0.0.1 9102 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key +127.0.0.1 9104 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key +127.0.0.1 9106 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key \ No newline at end of file diff --git a/test/deploy.sh b/test/deploy.sh index 6bbb12eb9b..5a949637e9 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -69,7 +69,7 @@ function launch_localnet() { if ${VERBOSE}; then verbosity=5 else - verbosity=3 + verbosity=5 fi base_args=(--log_folder "${log_folder}" --min_peers "${MIN}" --bootnodes "${BN_MA}" "--network_type=$NETWORK" --blspass file:"${ROOT}/.hmy/blspass.txt" "--dns=false" "--verbosity=${verbosity}" "--p2p.security.max-conn-per-ip=100") @@ -80,8 +80,11 @@ function launch_localnet() { while IFS='' read -r line || [[ -n "$line" ]]; do i=$((i + 1)) + + # Read config for i-th node form config file IFS=' ' read -r ip port mode bls_key shard node_config <<<"${line}" + echo "LINE: ${line} ${shard}" args=("${base_args[@]}" --ip "${ip}" --port "${port}" --key "/tmp/${ip}-${port}.key" --db_dir "${ROOT}/db-${ip}-${port}" "--broadcast_invalid_tx=false") if [[ -z "$ip" || -z "$port" ]]; then echo "skip empty node" From 89653714829d26ae98cf4eaddd98972131943fba Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 1 Sep 2022 01:00:51 +0700 Subject: [PATCH 003/420] in progress. --- consensus/consensus_v2.go | 1 - 1 file changed, 1 deletion(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 16554d0393..16fa18d565 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -131,7 +131,6 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb } func (consensus *Consensus) finalCommit() { - // THIS IS NOT GOOD PLACE FOR LEADER SWITCHING numCommits := consensus.Decider.SignersCount(quorum.Commit) consensus.getLogger().Info(). From 4d7e167bee342f9be7d2f79b882bf02fedbe46b8 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 1 Sep 2022 01:02:36 +0700 Subject: [PATCH 004/420] in progress. --- consensus/consensus_v2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 16fa18d565..9ab924c004 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -689,7 +689,7 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess // SetupForNewConsensus sets the state for new consensus func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) - curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) // first view id is going to be 2. + curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) prev := consensus.GetLeaderPubKey() idx := consensus.SetLeaderIndex(func(i int) int { if curBlockViewID%3 == 0 { From 0e1a1667e2f0013229862b1ddaab4fc338c50491 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 5 Sep 2022 19:39:49 +0700 Subject: [PATCH 005/420] consensus check is forked --- api/service/explorer/service.go | 6 ++-- consensus/consensus.go | 2 -- consensus/consensus_service.go | 14 -------- consensus/consensus_v2.go | 63 +++++++++++++++++++++++---------- internal/params/config.go | 10 ++++++ 5 files changed, 58 insertions(+), 37 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index d96c1bc56c..5887bb3a02 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -223,9 +223,9 @@ func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { for i := cur; i > 0; i-- { block := s.blockchain.GetBlockByNumber(i) - w.Write([]byte(fmt.Sprintf("%d ", i))) - w.Write([]byte(fmt.Sprintf("%s ", block.Header().ViewID().String()))) - w.Write([]byte(fmt.Sprintf("%s ", block.Header().Coinbase().Hash().Hex()))) + w.Write([]byte(fmt.Sprintf("#%d ", i))) + w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) + w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) } } diff --git a/consensus/consensus.go b/consensus/consensus.go index 2598010102..fb7bf79b53 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -73,8 +73,6 @@ type Consensus struct { priKey multibls.PrivateKeys // the publickey of leader LeaderPubKey *bls.PublicKeyWrapper - // index of leader in the list of validators. - LeaderIndex int // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e646504dde..d26ea8ced5 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -474,20 +474,6 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } -// SetLeaderIndex set the leader index. -func (consensus *Consensus) SetLeaderIndex(f func(int) int) (current int) { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() - consensus.LeaderIndex = f(consensus.LeaderIndex) - return consensus.LeaderIndex -} - -func (consensus *Consensus) GetLeaderIndex() int { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() - return consensus.LeaderIndex -} - // SetViewChangingID set the current view change ID func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 9ab924c004..a36d3d9212 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -5,6 +5,7 @@ import ( "context" "encoding/hex" "fmt" + "math/big" "sync/atomic" "time" @@ -691,27 +692,42 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) prev := consensus.GetLeaderPubKey() - idx := consensus.SetLeaderIndex(func(i int) int { - if curBlockViewID%3 == 0 { - return i + 1 + if consensus.Blockchain.Config().IsLeaderRotation(blk.Epoch()) { + epochBlockViewID, err := consensus.getEpochFirstBlockViewID(blk.Epoch()) + if err != nil { + consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", blk.Epoch().Uint64()) + return + } + if epochBlockViewID > curBlockViewID { + consensus.getLogger().Error().Msg("[SetupForNewConsensus] Epoch block viewID is greater than current block viewID") + return } - return i - }) - pps := consensus.Decider.Participants() - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = &pps[idx%len(pps)] - fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %q\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) - consensus.pubKeyLock.Unlock() - if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { - // leader changed - go func() { - fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - consensus.ReadySignal <- SyncProposal - - }() + diff := curBlockViewID - epochBlockViewID + + pps := consensus.Decider.Participants() + idx := (int(diff) / 3) % len(pps) + consensus.pubKeyLock.Lock() + fmt.Println("(int(diff)/3)%len(pps) == ", idx) + consensus.LeaderPubKey = &pps[idx] + fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + consensus.pubKeyLock.Unlock() + if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { + // leader changed + go func() { + fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + consensus.ReadySignal <- SyncProposal + + }() + } + } else { + fmt.Printf("SetupForNewConsensus0 :%d future v%d new: %s prev: %s %v\n", utils.GetPort(), curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] + consensus.pubKeyLock.Unlock() } + // Update consensus keys at last so the change of leader status doesn't mess up normal flow if blk.IsLastBlockInEpoch() { consensus.SetMode(consensus.UpdateConsensusInformation()) @@ -720,6 +736,17 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.ResetState() } +func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, error) { + if epoch.Uint64() == 0 { + return 0, nil + } + epochBlock := consensus.Blockchain.GetBlockByNumber(epoch.Uint64() - 1) + if epochBlock == nil { + return 0, errors.Errorf("block not found for number %d", epoch.Uint64()-1) + } + return epochBlock.Header().ViewID().Uint64() + 1, nil +} + func (consensus *Consensus) postCatchup(initBN uint64) { if initBN < consensus.BlockNum() { consensus.getLogger().Info(). diff --git a/internal/params/config.go b/internal/params/config.go index d24f4be7b4..9e9a3b4fc5 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -262,6 +262,7 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, FeeCollectEpoch: big.NewInt(5), + LeaderRotationEpoch: big.NewInt(1), } // AllProtocolChanges ... @@ -301,6 +302,7 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch + big.NewInt(1), // LeaderRotationEpoch big.NewInt(0), // FeeCollectEpoch } @@ -341,6 +343,8 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch + // TODO place correct epoch number + big.NewInt(1), // LeaderRotationEpoch big.NewInt(0), // FeeCollectEpoch } @@ -481,6 +485,8 @@ type ChainConfig struct { // AllowlistEpoch is the first epoch to support allowlist of HIP18 AllowlistEpoch *big.Int + LeaderRotationEpoch *big.Int `json:"leader-rotation-epoch,omitempty"` + // FeeCollectEpoch is the first epoch that enables txn fees to be collected into the community-managed account. // It should >= StakingEpoch. // Before StakingEpoch, txn fees are paid to miner/leader. @@ -687,6 +693,10 @@ func (c *ChainConfig) IsAllowlistEpoch(epoch *big.Int) bool { return isForked(c.AllowlistEpoch, epoch) } +func (c *ChainConfig) IsLeaderRotation(epoch *big.Int) bool { + return isForked(c.LeaderRotationEpoch, epoch) +} + // IsFeeCollectEpoch determines whether Txn Fees will be collected into the community-managed account. func (c *ChainConfig) IsFeeCollectEpoch(epoch *big.Int) bool { return isForked(c.FeeCollectEpoch, epoch) From 6ecf222f36358e5e9879a7289e8e89639671e3d6 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 5 Sep 2022 22:41:06 +0700 Subject: [PATCH 006/420] update master --- api/service/explorer/service.go | 2 ++ consensus/consensus_v2.go | 9 +++------ consensus/view_change.go | 2 +- node/node_newblock.go | 1 - 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 5887bb3a02..1ebfa94ba1 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -223,6 +223,8 @@ func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { for i := cur; i > 0; i-- { block := s.blockchain.GetBlockByNumber(i) + leaderPubKey, _ := chain.GetLeaderPubKeyFromCoinbase(s.backend.Blockchain(), block.Header()) + w.Write([]byte(fmt.Sprintf("%s ", leaderPubKey.Bytes.Hex()))) w.Write([]byte(fmt.Sprintf("#%d ", i))) w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index a36d3d9212..cdd641f906 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -708,21 +708,18 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg pps := consensus.Decider.Participants() idx := (int(diff) / 3) % len(pps) consensus.pubKeyLock.Lock() - fmt.Println("(int(diff)/3)%len(pps) == ", idx) + //fmt.Println("(int(diff)/3)%len(pps) == ", idx) consensus.LeaderPubKey = &pps[idx] - fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + //fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) consensus.pubKeyLock.Unlock() if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { // leader changed go func() { - fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) consensus.ReadySignal <- SyncProposal - }() } } else { - fmt.Printf("SetupForNewConsensus0 :%d future v%d new: %s prev: %s %v\n", utils.GetPort(), curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + fmt.Printf("SetupForNewConsensus0 Before LeaderRotation") consensus.pubKeyLock.Lock() consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] consensus.pubKeyLock.Unlock() diff --git a/consensus/view_change.go b/consensus/view_change.go index 2936fba018..9c045a9eef 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -257,7 +257,7 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - fmt.Printf("Message to send leader111: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) + fmt.Printf("[startViewChange]: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } diff --git a/node/node_newblock.go b/node/node_newblock.go index 24e26ad99a..66bd4ed038 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -148,7 +148,6 @@ func (node *Node) ProposeNewBlock(commitSigs chan []byte) (*types.Block, error) if node.Blockchain().Config().IsStaking(header.Epoch()) { blsPubKeyBytes := leaderKey.Object.GetAddress() coinbase.SetBytes(blsPubKeyBytes[:]) - fmt.Println("coinbase.SetBytes leader: ", leaderKey.Bytes.Hex(), coinbase.Hex()) } emptyAddr := common.Address{} From 35b72d5c1f510226d1c49c3640939a5d43aef27c Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 6 Sep 2022 02:58:48 +0700 Subject: [PATCH 007/420] fix leader --- consensus/consensus.go | 1 + consensus/consensus_service.go | 45 +++++++++++++++++++----- consensus/consensus_v2.go | 64 ++++++++++++++++++++++++++++------ internal/chain/engine.go | 2 +- 4 files changed, 91 insertions(+), 21 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index fb7bf79b53..f2f995ae7d 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -76,6 +76,7 @@ type Consensus struct { // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 + epoch uint64 // Blockhash - 32 byte blockHash [32]byte // Block to run consensus on diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index d26ea8ced5..d2747d19d9 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -77,6 +77,7 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) + consensus.pubKeyLock.Unlock() consensus.getLogger().Info().Msg("My Committee updated") for i := range pubKeys { consensus.getLogger().Info(). @@ -84,17 +85,24 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi Str("BLSPubKey", pubKeys[i].Bytes.Hex()). Msg("Member") } - - allKeys := consensus.Decider.Participants() - if len(allKeys) != 0 { - consensus.LeaderPubKey = &allKeys[0] - consensus.getLogger().Info(). - Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") + if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { + consensus.updateLeader() } else { - consensus.getLogger().Error(). - Msg("[UpdatePublicKeys] Participants is empty") + consensus.pubKeyLock.Lock() + allKeys := consensus.Decider.Participants() + consensus.pubKeyLock.Unlock() + if len(allKeys) != 0 { + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = &allKeys[0] + consensus.pubKeyLock.Unlock() + consensus.getLogger().Info(). + Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") + } else { + consensus.getLogger().Error(). + Msg("[UpdatePublicKeys] Participants is empty") + } } - consensus.pubKeyLock.Unlock() + // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. consensus.UpdateBitmaps() @@ -474,6 +482,15 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } +func (consensus *Consensus) SetCurEpoch(epoch uint64) { + fmt.Println("SetCurEpoch", epoch) + atomic.StoreUint64(&consensus.epoch, epoch) +} + +func (consensus *Consensus) GetCurEpoch() *big.Int { + return big.NewInt(int64(atomic.LoadUint64(&consensus.epoch))) +} + // SetViewChangingID set the current view change ID func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) @@ -608,3 +625,13 @@ func (consensus *Consensus) getLogger() *zerolog.Logger { Logger() return &logger } + +func UpdatePublicKeyDefault(consensus *Consensus) { + if allKeys := consensus.Decider.Participants(); len(allKeys) > 0 { + consensus.LeaderPubKey = &allKeys[0] + } +} + +func UpdatePublicKeyRotate(consensus *Consensus) { + //consensus +} diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index cdd641f906..1313f279be 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -11,7 +11,6 @@ import ( bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" - nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" @@ -687,15 +686,14 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess return nil } -// SetupForNewConsensus sets the state for new consensus -func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { - atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) - curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) +func (consensus *Consensus) updateLeader() { + curBlockViewID := consensus.GetCurBlockViewID() prev := consensus.GetLeaderPubKey() - if consensus.Blockchain.Config().IsLeaderRotation(blk.Epoch()) { - epochBlockViewID, err := consensus.getEpochFirstBlockViewID(blk.Epoch()) + epoch := consensus.GetCurEpoch() + if consensus.Blockchain.Config().IsLeaderRotation(epoch) { + epochBlockViewID, err := consensus.getEpochFirstBlockViewID(epoch) if err != nil { - consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", blk.Epoch().Uint64()) + consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", epoch) return } if epochBlockViewID > curBlockViewID { @@ -706,7 +704,8 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg diff := curBlockViewID - epochBlockViewID pps := consensus.Decider.Participants() - idx := (int(diff) / 3) % len(pps) + fmt.Println("diff ", diff, "epochBlockViewID: ", curBlockViewID, "epochBlockViewID", epochBlockViewID, (int(diff) / 5), len(pps), epoch) + idx := (int(diff) / 5) % len(pps) consensus.pubKeyLock.Lock() //fmt.Println("(int(diff)/3)%len(pps) == ", idx) consensus.LeaderPubKey = &pps[idx] @@ -718,8 +717,51 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.ReadySignal <- SyncProposal }() } + } +} + +// SetupForNewConsensus sets the state for new consensus +func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { + atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) + consensus.SetCurBlockViewID(committedMsg.ViewID + 1) + if blk.IsLastBlockInEpoch() { + consensus.SetCurEpoch(blk.Epoch().Uint64() + 1) + } else { + consensus.SetCurEpoch(blk.Epoch().Uint64()) + } + //prev := consensus.GetLeaderPubKey() + if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { + //consensus.updateLeader() + consensus.updateLeader() + + /*{ + epochBlockViewID, err := consensus.getEpochFirstBlockViewID(consensus.GetCurEpoch()) + if err != nil { + consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", blk.Epoch().Uint64()) + return + } + if epochBlockViewID > curBlockViewID { + consensus.getLogger().Error().Msg("[SetupForNewConsensus] Epoch block viewID is greater than current block viewID") + return + } + + diff := curBlockViewID - epochBlockViewID + + pps := consensus.Decider.Participants() + idx := (int(diff) / 3) % len(pps) + consensus.pubKeyLock.Lock() + //fmt.Println("(int(diff)/3)%len(pps) == ", idx) + consensus.LeaderPubKey = &pps[idx] + //fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + consensus.pubKeyLock.Unlock() + if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { + // leader changed + go func() { + consensus.ReadySignal <- SyncProposal + }() + } + }*/ } else { - fmt.Printf("SetupForNewConsensus0 Before LeaderRotation") consensus.pubKeyLock.Lock() consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] consensus.pubKeyLock.Unlock() @@ -737,7 +779,7 @@ func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, er if epoch.Uint64() == 0 { return 0, nil } - epochBlock := consensus.Blockchain.GetBlockByNumber(epoch.Uint64() - 1) + epochBlock := consensus.Blockchain.GetBlockByNumber(shard.Schedule.EpochLastBlock(epoch.Uint64() - 1)) if epochBlock == nil { return 0, errors.Errorf("block not found for number %d", epoch.Uint64()-1) } diff --git a/internal/chain/engine.go b/internal/chain/engine.go index a340e9c278..6e018dccfd 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -186,7 +186,7 @@ func (e *engineImpl) VerifyVRF( return nil } -// retrieve corresponding blsPublicKey from Coinbase Address +// GetLeaderPubKeyFromCoinbase retrieve corresponding blsPublicKey from Coinbase Address func GetLeaderPubKeyFromCoinbase( blockchain engine.ChainReader, h *block.Header, ) (*bls.PublicKeyWrapper, error) { From 91ecc7367be5f3fc7b47f646d3eec1775146cee4 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 7 Sep 2022 22:20:05 +0700 Subject: [PATCH 008/420] check leader for N blocks --- consensus/consensus.go | 5 +++ consensus/consensus_service.go | 29 +++++++-------- consensus/consensus_v2.go | 66 ++++++++++++++++++++-------------- consensus/view_change.go | 2 +- 4 files changed, 59 insertions(+), 43 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index f2f995ae7d..f9b48be2ab 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -181,6 +181,11 @@ func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper { defer consensus.pubKeyLock.Unlock() return consensus.LeaderPubKey } +func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = pub + consensus.pubKeyLock.Unlock() +} func (consensus *Consensus) GetPrivateKeys() multibls.PrivateKeys { return consensus.priKey diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index d2747d19d9..0d7a6c5543 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -77,8 +77,20 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) + allKeys := consensus.Decider.Participants() consensus.pubKeyLock.Unlock() - consensus.getLogger().Info().Msg("My Committee updated") + if len(allKeys) != 0 { + first := consensus.Decider.FirstParticipant( + shard.Schedule.InstanceForEpoch(consensus.Blockchain.CurrentHeader().Epoch())) + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = first + consensus.pubKeyLock.Unlock() + consensus.getLogger().Info(). + Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") + } else { + consensus.getLogger().Error(). + Msg("[UpdatePublicKeys] Participants is empty") + } for i := range pubKeys { consensus.getLogger().Info(). Int("index", i). @@ -87,20 +99,6 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi } if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { consensus.updateLeader() - } else { - consensus.pubKeyLock.Lock() - allKeys := consensus.Decider.Participants() - consensus.pubKeyLock.Unlock() - if len(allKeys) != 0 { - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = &allKeys[0] - consensus.pubKeyLock.Unlock() - consensus.getLogger().Info(). - Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") - } else { - consensus.getLogger().Error(). - Msg("[UpdatePublicKeys] Participants is empty") - } } // reset states after update public keys @@ -483,7 +481,6 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { } func (consensus *Consensus) SetCurEpoch(epoch uint64) { - fmt.Println("SetCurEpoch", epoch) atomic.StoreUint64(&consensus.epoch, epoch) } diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 1313f279be..d3e8dc8eb7 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -4,13 +4,13 @@ import ( "bytes" "context" "encoding/hex" - "fmt" "math/big" "sync/atomic" "time" bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" + "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" @@ -393,7 +393,6 @@ func (consensus *Consensus) Start( consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() case newBlock := <-blockChannel: - //consensus.ReshardingNextLeader(newBlock) consensus.getLogger().Info(). Uint64("MsgBlockNum", newBlock.NumberU64()). Msg("[ConsensusMainLoop] Received Proposed New Block!") @@ -405,7 +404,6 @@ func (consensus *Consensus) Start( } // Sleep to wait for the full block time consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") - <-time.After(time.Until(consensus.NextBlockDue)) consensus.StartFinalityCount() @@ -530,7 +528,6 @@ func (consensus *Consensus) getLastMileBlocksAndMsg(bnStart uint64) ([]*types.Bl // preCommitAndPropose commit the current block with 67% commit signatures and start // proposing new block which will wait on the full commit signatures to finish func (consensus *Consensus) preCommitAndPropose(blk *types.Block) error { - //fmt.Println("preCommitAndPropose", utils.GetPort(), blk.NumberU64()) if blk == nil { return errors.New("block to pre-commit is nil") } @@ -687,30 +684,48 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess } func (consensus *Consensus) updateLeader() { - curBlockViewID := consensus.GetCurBlockViewID() prev := consensus.GetLeaderPubKey() epoch := consensus.GetCurEpoch() + curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() if consensus.Blockchain.Config().IsLeaderRotation(epoch) { - epochBlockViewID, err := consensus.getEpochFirstBlockViewID(epoch) - if err != nil { - consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", epoch) - return + leader := consensus.GetLeaderPubKey() + for i := uint64(0); i < 5; i++ { + header := consensus.Blockchain.GetHeaderByNumber(curNumber - i) + if header == nil { + return + } + // Previous epoch, we should not change leader. + if header.Epoch().Uint64() != epoch.Uint64() { + return + } + // Check if the same leader. + pub, err := chain.GetLeaderPubKeyFromCoinbase(consensus.Blockchain, header) + if err != nil { + utils.Logger().Error().Err(err).Msg("Failed to get leader public key from coinbase") + return + } + if !pub.Object.IsEqual(leader.Object) { + // Another leader. + return + } + } + // Passed all checks, we can change leader. + var ( + wasFound bool + next *bls.PublicKeyWrapper + ) + // The same leader for N blocks. + if consensus.Blockchain.Config().IsAllowlistEpoch(epoch) { + wasFound, next = consensus.Decider.NthNextHmyExt(shard.Schedule.InstanceForEpoch(epoch), leader, 1) + } else { + wasFound, next = consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) } - if epochBlockViewID > curBlockViewID { - consensus.getLogger().Error().Msg("[SetupForNewConsensus] Epoch block viewID is greater than current block viewID") + if !wasFound { + utils.Logger().Error().Msg("Failed to get next leader") return + } else { + consensus.SetLeaderPubKey(next) } - - diff := curBlockViewID - epochBlockViewID - - pps := consensus.Decider.Participants() - fmt.Println("diff ", diff, "epochBlockViewID: ", curBlockViewID, "epochBlockViewID", epochBlockViewID, (int(diff) / 5), len(pps), epoch) - idx := (int(diff) / 5) % len(pps) - consensus.pubKeyLock.Lock() - //fmt.Println("(int(diff)/3)%len(pps) == ", idx) - consensus.LeaderPubKey = &pps[idx] - //fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) - consensus.pubKeyLock.Unlock() if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { // leader changed go func() { @@ -729,6 +744,9 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg } else { consensus.SetCurEpoch(blk.Epoch().Uint64()) } + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] + consensus.pubKeyLock.Unlock() //prev := consensus.GetLeaderPubKey() if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { //consensus.updateLeader() @@ -761,10 +779,6 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg }() } }*/ - } else { - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] - consensus.pubKeyLock.Unlock() } // Update consensus keys at last so the change of leader status doesn't mess up normal flow diff --git a/consensus/view_change.go b/consensus/view_change.go index 9c045a9eef..55d3d56a8b 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -176,7 +176,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe cur := consensus.GetCurBlockViewID() if viewID > cur { - gap = int(viewID - consensus.GetCurBlockViewID()) + gap = int(viewID - cur) } var lastLeaderPubKey *bls.PublicKeyWrapper var err error From 934ca73c5e90261ef2c3b94a523082340daf4538 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sat, 10 Sep 2022 18:02:45 +0700 Subject: [PATCH 009/420] fix --- api/service/explorer/service.go | 2 -- cmd/harmony/main.go | 3 --- consensus/consensus_service.go | 21 --------------------- consensus/quorum/quorum.go | 7 +------ 4 files changed, 1 insertion(+), 32 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 1ebfa94ba1..b13f73db61 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -115,8 +115,6 @@ func (s *Service) Run() *http.Server { s.router = mux.NewRouter() - fmt.Println("++", addr) - // Set up router for addresses. // Fetch addresses request, accepts parameter size: how much addresses to read, // parameter prefix: from which address prefix start diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 75eabfb2e8..1dabf5c459 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -265,8 +265,6 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { var err error - fmt.Println("OS: ", os.Args) - nodeconfigSetShardSchedule(hc) nodeconfig.SetShardingSchedule(shard.Schedule) nodeconfig.SetVersion(getHarmonyVersion()) @@ -787,7 +785,6 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // Set the consensus ID to be the current block number viewID := currentNode.Blockchain().CurrentBlock().Header().ViewID().Uint64() - fmt.Println("viewID:", viewID) currentConsensus.SetViewIDs(viewID + 1) utils.Logger().Info(). Uint64("viewID", viewID). diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 0d7a6c5543..96056b5231 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -219,7 +219,6 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } - fmt.Println("[checkViewID] Set LEADEER PUB KEY ", msg.SenderPubkeys[0].Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() @@ -403,7 +402,6 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { Str("leaderPubKey", leaderPubKey.Bytes.Hex()). Msg("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain") consensus.pubKeyLock.Lock() - fmt.Println("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain", leaderPubKey.Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = leaderPubKey consensus.pubKeyLock.Unlock() } @@ -498,15 +496,6 @@ func (consensus *Consensus) StartFinalityCount() { consensus.finalityCounter.Store(time.Now().UnixNano()) } -//func (consensus *Consensus) ReshardingNextLeader(newblock *types.Block) { -// consensus.pubKeyLock.Lock() -// fmt.Println("nextBlock1 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) -// consensus.LeaderPubKey = consensus.getNextLeaderKey(consensus.GetCurBlockViewID() + 1) -// fmt.Println("nextBlock2 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) -// consensus.pubKeyLock.Unlock() -// -//} - // FinishFinalityCount calculate the current finality func (consensus *Consensus) FinishFinalityCount() { d := time.Now().UnixNano() @@ -622,13 +611,3 @@ func (consensus *Consensus) getLogger() *zerolog.Logger { Logger() return &logger } - -func UpdatePublicKeyDefault(consensus *Consensus) { - if allKeys := consensus.Decider.Participants(); len(allKeys) > 0 { - consensus.LeaderPubKey = &allKeys[0] - } -} - -func UpdatePublicKeyRotate(consensus *Consensus) { - //consensus -} diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 7e76dfcc6b..867fd99677 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -231,17 +231,12 @@ func (s *cIdentities) NthNextHmy(instance shardingconfig.Instance, pubKey *bls.P Msg("[NthNextHmy] pubKey not found") } numNodes := instance.NumHarmonyOperatedNodesPerShard() - //fmt.Println("??idx:", idx, numNodes) // sanity check to avoid out of bound access if numNodes <= 0 || numNodes > len(s.publicKeys) { numNodes = len(s.publicKeys) } idx = (idx + next) % numNodes - //fmt.Println("-------idx:", idx) - - new := &s.publicKeys[idx] - fmt.Println("NthNextHmy: ", pubKey.Bytes.Hex(), new.Bytes.Hex()) - return found, new + return found, &s.publicKeys[idx] } // NthNextHmyExt return the Nth next pubkey of Harmony + allowlist nodes, next can be negative number From df9a60a7431a56308e6b47b8b0e20463a2e4f6c0 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 14:04:09 +0700 Subject: [PATCH 010/420] fix --- consensus/consensus_service.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 96056b5231..25594e044e 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync/atomic" "time" @@ -468,7 +467,6 @@ func (consensus *Consensus) isLeader() bool { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { - fmt.Println("SetViewIDs", height) consensus.SetCurBlockViewID(height) consensus.SetViewChangingID(height) } From f1b2f27c36e3c6b7a517a0a85ad326304523dcdf Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 15:27:56 +0700 Subject: [PATCH 011/420] Cleanup and fix update pub keys. --- consensus/consensus_service.go | 31 +++++++++--------- consensus/consensus_v2.go | 35 ++------------------- consensus/leader.go | 11 ++----- consensus/quorum/quorum.go | 1 - consensus/view_change.go | 15 +-------- p2p/stream/common/streammanager/cooldown.go | 6 ++++ 6 files changed, 27 insertions(+), 72 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 25594e044e..77996258e4 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync/atomic" "time" @@ -73,33 +74,31 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { + if utils.GetPort() == 9000 { + //utils.Logger().Info().Msg("UpdatePublicKeys") + fmt.Println("UpdatePublicKeys", len(pubKeys), len(allowlist)) + } // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) + consensus.getLogger().Info().Msg("My Committee updated") + for i := range pubKeys { + consensus.getLogger().Info(). + Int("index", i). + Str("BLSPubKey", pubKeys[i].Bytes.Hex()). + Msg("Member") + } + allKeys := consensus.Decider.Participants() - consensus.pubKeyLock.Unlock() if len(allKeys) != 0 { - first := consensus.Decider.FirstParticipant( - shard.Schedule.InstanceForEpoch(consensus.Blockchain.CurrentHeader().Epoch())) - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = first - consensus.pubKeyLock.Unlock() + consensus.LeaderPubKey = &allKeys[0] consensus.getLogger().Info(). Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") } else { consensus.getLogger().Error(). Msg("[UpdatePublicKeys] Participants is empty") } - for i := range pubKeys { - consensus.getLogger().Info(). - Int("index", i). - Str("BLSPubKey", pubKeys[i].Bytes.Hex()). - Msg("Member") - } - if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { - consensus.updateLeader() - } - + consensus.pubKeyLock.Unlock() // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. consensus.UpdateBitmaps() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index d3e8dc8eb7..fe685ca857 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -645,7 +645,6 @@ func (consensus *Consensus) tryCatchup() error { consensus.getLogger().Error().Err(err).Msg("[TryCatchup] Failed to add block to chain") return err } - //fmt.Println("tryCatchup ", utils.GetPort(), blk.NumberU64()) select { // TODO: Remove this when removing dns sync and stream sync is fully up case consensus.VerifiedNewBlock <- blk: @@ -683,7 +682,7 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess return nil } -func (consensus *Consensus) updateLeader() { +func (consensus *Consensus) rotateLeader() { prev := consensus.GetLeaderPubKey() epoch := consensus.GetCurEpoch() curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() @@ -747,38 +746,8 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.pubKeyLock.Lock() consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] consensus.pubKeyLock.Unlock() - //prev := consensus.GetLeaderPubKey() if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { - //consensus.updateLeader() - consensus.updateLeader() - - /*{ - epochBlockViewID, err := consensus.getEpochFirstBlockViewID(consensus.GetCurEpoch()) - if err != nil { - consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", blk.Epoch().Uint64()) - return - } - if epochBlockViewID > curBlockViewID { - consensus.getLogger().Error().Msg("[SetupForNewConsensus] Epoch block viewID is greater than current block viewID") - return - } - - diff := curBlockViewID - epochBlockViewID - - pps := consensus.Decider.Participants() - idx := (int(diff) / 3) % len(pps) - consensus.pubKeyLock.Lock() - //fmt.Println("(int(diff)/3)%len(pps) == ", idx) - consensus.LeaderPubKey = &pps[idx] - //fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) - consensus.pubKeyLock.Unlock() - if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { - // leader changed - go func() { - consensus.ReadySignal <- SyncProposal - }() - } - }*/ + consensus.rotateLeader() } // Update consensus keys at last so the change of leader status doesn't mess up normal flow diff --git a/consensus/leader.go b/consensus/leader.go index a359c229a4..4f72ae6268 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -200,16 +200,12 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { // TODO HERE - //if recvMsg.ViewID == 10 { - // return - //} + if recvMsg.ViewID == 21 { + return + } consensus.mutex.Lock() defer consensus.mutex.Unlock() //// Read - Start - if consensus.ShardID == 0 { - //fmt.Println("onCommit ", recvMsg.BlockNum) - } - if !consensus.isRightBlockNumAndViewID(recvMsg) { return } @@ -341,5 +337,4 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED) } - //fmt.Println("onCommit99: ", utils.GetPort(), recvMsg.BlockNum) } diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 867fd99677..6cf79e549a 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -75,7 +75,6 @@ type ParticipantTracker interface { Participants() multibls.PublicKeys IndexOf(bls.SerializedPublicKey) int ParticipantsCount() int64 - NthNext(*bls.PublicKeyWrapper, int) (bool, *bls.PublicKeyWrapper) NthNextHmy(shardingconfig.Instance, *bls.PublicKeyWrapper, int) (bool, *bls.PublicKeyWrapper) NthNextHmyExt(shardingconfig.Instance, *bls.PublicKeyWrapper, int) (bool, *bls.PublicKeyWrapper) FirstParticipant(shardingconfig.Instance) *bls.PublicKeyWrapper diff --git a/consensus/view_change.go b/consensus/view_change.go index 55d3d56a8b..3bd0c181f2 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync" "time" @@ -162,7 +161,6 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { Uint64("stuckBlockViewID", stuckBlockViewID). Msg("[getNextViewID]") - fmt.Println("end getNextViewID: ", nextViewID, viewChangeDuration) // duration is always the fixed view change duration for synchronous view change return nextViewID, viewChangeDuration } @@ -235,7 +233,6 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe lastLeaderPubKey, gap) } - fmt.Println("wasfoundNext", consensus.Blockchain.Config().IsAllowlistEpoch(epoch), wasFound, next.Bytes.Hex(), lastLeaderPubKey.Bytes.Hex()) if !wasFound { consensus.getLogger().Warn(). Str("key", consensus.LeaderPubKey.Bytes.Hex()). @@ -257,7 +254,6 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - fmt.Printf("[startViewChange]: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } @@ -268,7 +264,6 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() - //fmt.Println("startViewChange", nextViewID) consensus.SetViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet @@ -279,8 +274,6 @@ func (consensus *Consensus) startViewChange() { consensus.pubKeyLock.Lock() lpk := consensus.getNextLeaderKey(nextViewID) consensus.LeaderPubKey = lpk - //fmt.Println("Message to send leader cur: ", consensus.LeaderPubKey.Bytes.Hex(), "next: ", lpk.Bytes.Hex()) - //fmt.Println("Message to send leader: ", consensus.LeaderPubKey.Bytes.Hex()) consensus.pubKeyLock.Unlock() consensus.getLogger().Warn(). @@ -314,9 +307,7 @@ func (consensus *Consensus) startViewChange() { if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { continue } - // Тут уже другой leader msgToSend := consensus.constructViewChangeMessage(&key) - fmt.Println("Message to send leader222: ", consensus.LeaderPubKey.Bytes.Hex()) if err := consensus.msgSender.SendWithRetry( consensus.BlockNum(), msg_pb.MessageType_VIEWCHANGE, @@ -374,8 +365,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.ResetState() } - fmt.Println("[startNewView]", newLeaderPriKey.Pub.Bytes.Hex()) - consensus.LeaderPubKey = newLeaderPriKey.Pub + consensus.SetLeaderPubKey(newLeaderPriKey.Pub) return nil } @@ -492,8 +482,6 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.mutex.Lock() defer consensus.mutex.Unlock() - fmt.Printf("[onNewView] received new view message from %+v\n", recvMsg) - consensus.getLogger().Info(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -582,7 +570,6 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // newView message verified success, override my state consensus.SetViewIDs(recvMsg.ViewID) consensus.pubKeyLock.Lock() - fmt.Println("[onNewView1221] new leader key cur:", consensus.LeaderPubKey.Bytes.Hex(), " new: ", senderKey.Bytes.Hex()) consensus.LeaderPubKey = senderKey consensus.pubKeyLock.Unlock() consensus.ResetViewChangeState() diff --git a/p2p/stream/common/streammanager/cooldown.go b/p2p/stream/common/streammanager/cooldown.go index 0f837c01de..d8b58346c0 100644 --- a/p2p/stream/common/streammanager/cooldown.go +++ b/p2p/stream/common/streammanager/cooldown.go @@ -2,6 +2,7 @@ package streammanager import ( "container/list" + "sync" "time" "github.com/libp2p/go-libp2p/core/peer" @@ -14,6 +15,7 @@ const ( ) type coolDownCache struct { + mu sync.Mutex timeCache *timecache.TimeCache } @@ -26,6 +28,8 @@ func newCoolDownCache() *coolDownCache { // Has check and add the peer ID to the cache func (cache *coolDownCache) Has(id peer.ID) bool { + cache.mu.Lock() + defer cache.mu.Unlock() has := cache.timeCache.Has(string(id)) if !has { cache.timeCache.Add(string(id)) @@ -35,6 +39,8 @@ func (cache *coolDownCache) Has(id peer.ID) bool { // Reset the cool down cache func (cache *coolDownCache) Reset() { + cache.mu.Lock() + defer cache.mu.Unlock() cache.timeCache.Q = list.New() cache.timeCache.M = make(map[string]time.Time) } From 26dbb8d9df0778142c4a5b978efae12cb70a49e0 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 16:32:36 +0700 Subject: [PATCH 012/420] Rotate leader. --- consensus/consensus.go | 1 - consensus/consensus_service.go | 8 -------- consensus/consensus_v2.go | 12 +++--------- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index f9b48be2ab..1259576834 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -76,7 +76,6 @@ type Consensus struct { // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 - epoch uint64 // Blockhash - 32 byte blockHash [32]byte // Block to run consensus on diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 77996258e4..af344c6820 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -475,14 +475,6 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } -func (consensus *Consensus) SetCurEpoch(epoch uint64) { - atomic.StoreUint64(&consensus.epoch, epoch) -} - -func (consensus *Consensus) GetCurEpoch() *big.Int { - return big.NewInt(int64(atomic.LoadUint64(&consensus.epoch))) -} - // SetViewChangingID set the current view change ID func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index fe685ca857..7c4290fad2 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -682,9 +682,8 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess return nil } -func (consensus *Consensus) rotateLeader() { +func (consensus *Consensus) rotateLeader(epoch *big.Int) { prev := consensus.GetLeaderPubKey() - epoch := consensus.GetCurEpoch() curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() if consensus.Blockchain.Config().IsLeaderRotation(epoch) { leader := consensus.GetLeaderPubKey() @@ -738,16 +737,11 @@ func (consensus *Consensus) rotateLeader() { func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) consensus.SetCurBlockViewID(committedMsg.ViewID + 1) - if blk.IsLastBlockInEpoch() { - consensus.SetCurEpoch(blk.Epoch().Uint64() + 1) - } else { - consensus.SetCurEpoch(blk.Epoch().Uint64()) - } consensus.pubKeyLock.Lock() consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] consensus.pubKeyLock.Unlock() - if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { - consensus.rotateLeader() + if consensus.Blockchain.Config().IsLeaderRotation(blk.Epoch()) { + consensus.rotateLeader(blk.Epoch()) } // Update consensus keys at last so the change of leader status doesn't mess up normal flow From 6d6e37b792e1f660b2d28fccc8bb65434859d4e2 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 13 Sep 2022 01:10:13 +0700 Subject: [PATCH 013/420] fix fix fix fix fix --- consensus/consensus_service.go | 5 ----- consensus/consensus_v2.go | 16 ++++++++++++---- consensus/leader.go | 4 ---- internal/params/config.go | 7 ++++++- node/node_newblock.go | 2 +- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index af344c6820..e6be606f2e 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync/atomic" "time" @@ -74,10 +73,6 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { - if utils.GetPort() == 9000 { - //utils.Logger().Info().Msg("UpdatePublicKeys") - fmt.Println("UpdatePublicKeys", len(pubKeys), len(allowlist)) - } // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 7c4290fad2..3122a368b9 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -8,6 +8,7 @@ import ( "sync/atomic" "time" + "github.com/ethereum/go-ethereum/common" bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" "github.com/harmony-one/harmony/internal/chain" @@ -685,10 +686,11 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess func (consensus *Consensus) rotateLeader(epoch *big.Int) { prev := consensus.GetLeaderPubKey() curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() + utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), consensus.Blockchain.Config().IsLeaderRotation(epoch), consensus.Blockchain.Config().LeaderRotationBlocksCount) if consensus.Blockchain.Config().IsLeaderRotation(epoch) { leader := consensus.GetLeaderPubKey() - for i := uint64(0); i < 5; i++ { - header := consensus.Blockchain.GetHeaderByNumber(curNumber - i) + for i := 0; i < consensus.Blockchain.Config().LeaderRotationBlocksCount; i++ { + header := consensus.Blockchain.GetHeaderByNumber(curNumber - uint64(i)) if header == nil { return } @@ -740,8 +742,14 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.pubKeyLock.Lock() consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] consensus.pubKeyLock.Unlock() - if consensus.Blockchain.Config().IsLeaderRotation(blk.Epoch()) { - consensus.rotateLeader(blk.Epoch()) + var epoch *big.Int + if blk.IsLastBlockInEpoch() { + epoch = new(big.Int).Add(blk.Epoch(), common.Big1) + } else { + epoch = blk.Epoch() + } + if consensus.Blockchain.Config().IsLeaderRotation(epoch) { + consensus.rotateLeader(epoch) } // Update consensus keys at last so the change of leader status doesn't mess up normal flow diff --git a/consensus/leader.go b/consensus/leader.go index 4f72ae6268..422508762b 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -199,10 +199,6 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { - // TODO HERE - if recvMsg.ViewID == 21 { - return - } consensus.mutex.Lock() defer consensus.mutex.Unlock() //// Read - Start diff --git a/internal/params/config.go b/internal/params/config.go index 9e9a3b4fc5..55e8126e80 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -262,7 +262,8 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, FeeCollectEpoch: big.NewInt(5), - LeaderRotationEpoch: big.NewInt(1), + LeaderRotationEpoch: big.NewInt(4), + LeaderRotationBlocksCount: 5, } // AllProtocolChanges ... @@ -303,6 +304,7 @@ var ( big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } @@ -345,6 +347,7 @@ var ( big.NewInt(0), // AllowlistEpoch // TODO place correct epoch number big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } @@ -487,6 +490,8 @@ type ChainConfig struct { LeaderRotationEpoch *big.Int `json:"leader-rotation-epoch,omitempty"` + LeaderRotationBlocksCount int `json:"leader-rotation-blocks-count,omitempty"` + // FeeCollectEpoch is the first epoch that enables txn fees to be collected into the community-managed account. // It should >= StakingEpoch. // Before StakingEpoch, txn fees are paid to miner/leader. diff --git a/node/node_newblock.go b/node/node_newblock.go index 66bd4ed038..8738a98076 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -89,7 +89,7 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp newBlock, err := node.ProposeNewBlock(newCommitSigsChan) if err == nil { - fmt.Printf("ProposeNewBlock: #%d :%d @%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), newBlock.Header().ViewID().Int64(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) + fmt.Printf("ProposeNewBlock: #%d :%d e:%d, sh:%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), node.Blockchain().CurrentHeader().Epoch().Uint64(), node.Blockchain().ShardID(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). From 7de4b30e4474d0d42b99cf23735b9738789bb7d1 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 13 Sep 2022 02:15:25 +0700 Subject: [PATCH 014/420] Cleaned. --- api/service/explorer/service.go | 48 ------------------------------- cmd/harmony/main.go | 3 +- consensus/consensus_v2.go | 13 --------- consensus/validator.go | 14 --------- consensus/view_change.go | 8 ------ node/node_newblock.go | 3 -- test/configs/local-resharding.txt | 31 ++++++++++++++------ test/deploy.sh | 5 +--- 8 files changed, 24 insertions(+), 101 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index b13f73db61..b5ffa1b8b0 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -7,13 +7,11 @@ import ( "fmt" "net" "net/http" - "os" "path" "strconv" "time" "github.com/RoaringBitmap/roaring/roaring64" - "github.com/harmony-one/harmony/internal/common" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" ethCommon "github.com/ethereum/go-ethereum/common" @@ -121,9 +119,6 @@ func (s *Service) Run() *http.Server { s.router.Path("/addresses").Queries("size", "{[0-9]*?}", "prefix", "{[a-zA-Z0-9]*?}").HandlerFunc(s.GetAddresses).Methods("GET") s.router.Path("/addresses").HandlerFunc(s.GetAddresses) s.router.Path("/height").HandlerFunc(s.GetHeight) - s.router.Path("/leader").HandlerFunc(s.GetLeader) - s.router.Path("/blocks").HandlerFunc(s.GetBlocks) - s.router.Path("/halt").HandlerFunc(s.halt) // Set up router for supply info s.router.Path("/burn-addresses").Queries().HandlerFunc(s.GetInaccessibleAddressInfo).Methods("GET") @@ -191,49 +186,6 @@ type HeightResponse struct { S3 uint64 `json:"3,omitempty"` } -func (s *Service) GetLeader(w http.ResponseWriter, r *http.Request) { - if s.backend.IsCurrentlyLeader() { - w.Write([]byte("true ")) - } else { - w.Write([]byte("false")) - } - - keys := "" - for _, p := range s.backend.GetPublicKeys() { - addr := common.Address{} - addrBytes := p.Object.GetAddress() - addr.SetBytes(addrBytes[:]) - keys += fmt.Sprintf("%s ", addr.String()) - break - } - //blsPubKeyBytes := leaderKey.Object.GetAddress() - //coinbase.SetBytes(blsPubKeyBytes[:]) - - w.Write([]byte(fmt.Sprintf(" %d", s.blockchain.ShardID()))) - w.Write([]byte(fmt.Sprintf(" %s", s.Port))) - w.Write([]byte(fmt.Sprintf(" %s", keys))) - w.Write([]byte(fmt.Sprintf(" %s", s.backend.GetPublicKeys().SerializeToHexStr()))) - -} - -func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { - cur := s.blockchain.CurrentHeader().Number().Uint64() - - for i := cur; i > 0; i-- { - block := s.blockchain.GetBlockByNumber(i) - leaderPubKey, _ := chain.GetLeaderPubKeyFromCoinbase(s.backend.Blockchain(), block.Header()) - w.Write([]byte(fmt.Sprintf("%s ", leaderPubKey.Bytes.Hex()))) - w.Write([]byte(fmt.Sprintf("#%d ", i))) - w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) - w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) - w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) - } -} - -func (s *Service) halt(w http.ResponseWriter, r *http.Request) { - os.Exit(0) -} - // GetHeight returns heights of current and beacon chains if needed. func (s *Service) GetHeight(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 1dabf5c459..437a91dc59 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -420,9 +420,8 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { if currentNode.NodeConfig.Role() == nodeconfig.Validator { currentNode.RegisterValidatorServices() } else if currentNode.NodeConfig.Role() == nodeconfig.ExplorerNode { - + currentNode.RegisterExplorerServices() } - currentNode.RegisterExplorerServices() currentNode.RegisterService(service.CrosslinkSending, crosslink_sending.New(currentNode, currentNode.Blockchain())) if hc.Pprof.Enabled { setupPprofService(currentNode, hc) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 3122a368b9..e87ea21012 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -660,8 +660,6 @@ func (consensus *Consensus) tryCatchup() error { } func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMessage) error { - // this function evaluates for all, leader and validators. - if consensus.Blockchain().CurrentBlock().NumberU64() < blk.NumberU64() { if _, err := consensus.Blockchain().InsertChain([]*types.Block{blk}, !consensus.FBFTLog.IsBlockVerified(blk.Hash())); err != nil { consensus.getLogger().Error().Err(err).Msg("[commitBlock] Failed to add block to chain") @@ -760,17 +758,6 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.ResetState() } -func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, error) { - if epoch.Uint64() == 0 { - return 0, nil - } - epochBlock := consensus.Blockchain.GetBlockByNumber(shard.Schedule.EpochLastBlock(epoch.Uint64() - 1)) - if epochBlock == nil { - return 0, errors.Errorf("block not found for number %d", epoch.Uint64()-1) - } - return epochBlock.Header().ViewID().Uint64() + 1, nil -} - func (consensus *Consensus) postCatchup(initBN uint64) { if initBN < consensus.BlockNum() { consensus.getLogger().Info(). diff --git a/consensus/validator.go b/consensus/validator.go index 013769395c..a73ac92eb3 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -18,7 +18,6 @@ import ( ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { - recvMsg, err := consensus.ParseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). @@ -27,9 +26,6 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Msg("[OnAnnounce] Unparseable leader message") return } - if consensus.ShardID == 0 { - //fmt.Println("onAnnounce called ", recvMsg.BlockNum) - } // NOTE let it handle its own logs if !consensus.onAnnounceSanityChecks(recvMsg) { @@ -188,9 +184,6 @@ func (consensus *Consensus) sendCommitMessages(blockObj *types.Block) { // if onPrepared accepts the prepared message from the leader, then // it will send a COMMIT message for the leader to receive on the network. func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { - if consensus.ShardID == 0 { - //fmt.Println("onPrepared", recvMsg.BlockNum) - } consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -406,13 +399,6 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { consensus.getLogger().Info().Msg("[OnCommitted] Start consensus timer (new block added)") consensus.consensusTimeout[timeoutConsensus].Start() } - - //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum) - if blk != nil { - //consensus.ReshardingNextLeader(blk) - } else { - //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum, "blk is nil") - } } // Collect private keys that are part of the current committee. diff --git a/consensus/view_change.go b/consensus/view_change.go index 3bd0c181f2..6631051dcc 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -372,7 +372,6 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri // onViewChange is called when the view change message is received. func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { - //fmt.Printf("[onViewChange] received view change message from %+v\n", recvMsg) consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -395,13 +394,6 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { return } - consensus.getLogger().Debug(). - Err(err). - Interface("SenderPubkeys", recvMsg.SenderPubkeys). - Str("NextLeader", recvMsg.LeaderPubkey.Bytes.Hex()). - Str("myBLSPubKey", consensus.priKey.GetPublicKeys().SerializeToHexStr()). - Msg("[onViewChange] I am the Leader") - if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) { consensus.getLogger().Info(). Int64("have", consensus.Decider.SignersCount(quorum.ViewChange)). diff --git a/node/node_newblock.go b/node/node_newblock.go index 8738a98076..03fd69d9dd 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -2,7 +2,6 @@ package node import ( "errors" - "fmt" "sort" "strings" "time" @@ -89,8 +88,6 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp newBlock, err := node.ProposeNewBlock(newCommitSigsChan) if err == nil { - fmt.Printf("ProposeNewBlock: #%d :%d e:%d, sh:%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), node.Blockchain().CurrentHeader().Epoch().Uint64(), node.Blockchain().ShardID(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) - if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). Msg("Block with the same number was already proposed, abort.") diff --git a/test/configs/local-resharding.txt b/test/configs/local-resharding.txt index fc3d3e97b4..4ce91ece1c 100644 --- a/test/configs/local-resharding.txt +++ b/test/configs/local-resharding.txt @@ -1,12 +1,25 @@ 127.0.0.1 9000 validator .hmy/65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204.key -127.0.0.1 9002 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key -127.0.0.1 9004 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key -127.0.0.1 9006 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key -127.0.0.1 9008 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key -127.0.0.1 9010 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key -127.0.0.1 9099 explorer null 0 +127.0.0.1 9002 validator .hmy/40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809.key +127.0.0.1 9004 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key +127.0.0.1 9006 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key +127.0.0.1 9008 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key +127.0.0.1 9010 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key +127.0.0.1 9012 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key +127.0.0.1 9014 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key +127.0.0.1 9016 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key +127.0.0.1 9018 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key +127.0.0.1 9020 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key +127.0.0.1 9022 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key +127.0.0.1 9200 explorer null 0 127.0.0.1 9100 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key -127.0.0.1 9102 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key -127.0.0.1 9104 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key -127.0.0.1 9106 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key \ No newline at end of file +127.0.0.1 9102 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key +127.0.0.1 9104 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key +127.0.0.1 9106 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key +127.0.0.1 9108 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key +127.0.0.1 9110 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key +127.0.0.1 9112 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key +127.0.0.1 9114 validator .hmy/f47238daef97d60deedbde5302d05dea5de67608f11f406576e363661f7dcbc4a1385948549b31a6c70f6fde8a391486.key +127.0.0.1 9116 validator .hmy/fc4b9c535ee91f015efff3f32fbb9d32cdd9bfc8a837bb3eee89b8fff653c7af2050a4e147ebe5c7233dc2d5df06ee0a.key +127.0.0.1 9118 validator .hmy/ca86e551ee42adaaa6477322d7db869d3e203c00d7b86c82ebee629ad79cb6d57b8f3db28336778ec2180e56a8e07296.key +127.0.0.1 9300 explorer null 1 \ No newline at end of file diff --git a/test/deploy.sh b/test/deploy.sh index 5a949637e9..6bbb12eb9b 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -69,7 +69,7 @@ function launch_localnet() { if ${VERBOSE}; then verbosity=5 else - verbosity=5 + verbosity=3 fi base_args=(--log_folder "${log_folder}" --min_peers "${MIN}" --bootnodes "${BN_MA}" "--network_type=$NETWORK" --blspass file:"${ROOT}/.hmy/blspass.txt" "--dns=false" "--verbosity=${verbosity}" "--p2p.security.max-conn-per-ip=100") @@ -80,11 +80,8 @@ function launch_localnet() { while IFS='' read -r line || [[ -n "$line" ]]; do i=$((i + 1)) - - # Read config for i-th node form config file IFS=' ' read -r ip port mode bls_key shard node_config <<<"${line}" - echo "LINE: ${line} ${shard}" args=("${base_args[@]}" --ip "${ip}" --port "${port}" --key "/tmp/${ip}-${port}.key" --db_dir "${ROOT}/db-${ip}-${port}" "--broadcast_invalid_tx=false") if [[ -z "$ip" || -z "$port" ]]; then echo "skip empty node" From 50ed1c666b663c417a250914cb209382db072764 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 19 Sep 2022 21:40:21 +0700 Subject: [PATCH 015/420] Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. --- consensus/consensus_v2.go | 14 ++-------- core/blockchain.go | 3 ++ core/blockchain_impl.go | 59 +++++++++++++++++++++++++++++++++++++++ core/blockchain_stub.go | 5 ++++ 4 files changed, 69 insertions(+), 12 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index e87ea21012..5fe176c2f4 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -11,7 +11,6 @@ import ( "github.com/ethereum/go-ethereum/common" bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" - "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" @@ -697,7 +696,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { return } // Check if the same leader. - pub, err := chain.GetLeaderPubKeyFromCoinbase(consensus.Blockchain, header) + pub, err := consensus.Blockchain.GetLeaderPubKeyFromCoinbase(header) if err != nil { utils.Logger().Error().Err(err).Msg("Failed to get leader public key from coinbase") return @@ -708,16 +707,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { } } // Passed all checks, we can change leader. - var ( - wasFound bool - next *bls.PublicKeyWrapper - ) - // The same leader for N blocks. - if consensus.Blockchain.Config().IsAllowlistEpoch(epoch) { - wasFound, next = consensus.Decider.NthNextHmyExt(shard.Schedule.InstanceForEpoch(epoch), leader, 1) - } else { - wasFound, next = consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) - } + wasFound, next := consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) if !wasFound { utils.Logger().Error().Msg("Failed to get next leader") return diff --git a/core/blockchain.go b/core/blockchain.go index 1c7ea43d33..fda4831655 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -13,6 +13,7 @@ import ( "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" + "github.com/harmony-one/harmony/crypto/bls" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/tikv/redis_helper" @@ -334,6 +335,8 @@ type BlockChain interface { state *state.DB, ) (status WriteStatus, err error) + GetLeaderPubKeyFromCoinbase(h *block.Header) (*bls.PublicKeyWrapper, error) + // ========== Only For Tikv Start ========== // return true if is tikv writer master diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 73f37f8623..489fb3661e 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -39,6 +39,7 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" + bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/block" consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/consensus/reward" @@ -47,6 +48,7 @@ import ( "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" + "github.com/harmony-one/harmony/crypto/bls" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/tikv" @@ -181,6 +183,7 @@ type BlockChainImpl struct { validatorListByDelegatorCache *lru.Cache // Cache of validator list by delegator pendingCrossLinksCache *lru.Cache // Cache of last pending crosslinks blockAccumulatorCache *lru.Cache // Cache of block accumulators + leaderPubKeyFromCoinbase *lru.Cache // Cache of leader public key from coinbase quit chan struct{} // blockchain quit channel running int32 // running must be called atomically blockchainPruner *blockchainPruner // use to prune beacon chain @@ -242,6 +245,7 @@ func newBlockChainWithOptions( validatorListByDelegatorCache, _ := lru.New(validatorListByDelegatorCacheLimit) pendingCrossLinksCache, _ := lru.New(pendingCrossLinksCacheLimit) blockAccumulatorCache, _ := lru.New(blockAccumulatorCacheLimit) + leaderPubKeyFromCoinbase, _ := lru.New(chainConfig.LeaderRotationBlocksCount + 2) bc := &BlockChainImpl{ chainConfig: chainConfig, @@ -265,6 +269,7 @@ func newBlockChainWithOptions( validatorListByDelegatorCache: validatorListByDelegatorCache, pendingCrossLinksCache: pendingCrossLinksCache, blockAccumulatorCache: blockAccumulatorCache, + leaderPubKeyFromCoinbase: leaderPubKeyFromCoinbase, blockchainPruner: newBlockchainPruner(db), engine: engine, vmConfig: vmConfig, @@ -3256,6 +3261,60 @@ func (bc *BlockChainImpl) SuperCommitteeForNextEpoch( return nextCommittee, err } +// GetLeaderPubKeyFromCoinbase retrieve corresponding blsPublicKey from Coinbase Address +func (bc *BlockChainImpl) GetLeaderPubKeyFromCoinbase(h *block.Header) (*bls.PublicKeyWrapper, error) { + if cached, ok := bc.leaderPubKeyFromCoinbase.Get(h.Number().Uint64()); ok { + return cached.(*bls.PublicKeyWrapper), nil + } + rs, err := bc.getLeaderPubKeyFromCoinbase(h) + if err != nil { + return nil, err + } + bc.leaderPubKeyFromCoinbase.Add(h.Number().Uint64(), rs) + return rs, nil +} + +// getLeaderPubKeyFromCoinbase retrieve corresponding blsPublicKey from Coinbase Address +func (bc *BlockChainImpl) getLeaderPubKeyFromCoinbase(h *block.Header) (*bls.PublicKeyWrapper, error) { + shardState, err := bc.ReadShardState(h.Epoch()) + if err != nil { + return nil, errors.Wrapf(err, "cannot read shard state %v %s", + h.Epoch(), + h.Coinbase().Hash().Hex(), + ) + } + + committee, err := shardState.FindCommitteeByID(h.ShardID()) + if err != nil { + return nil, err + } + + committerKey := new(bls2.PublicKey) + isStaking := bc.Config().IsStaking(h.Epoch()) + for _, member := range committee.Slots { + if isStaking { + // After staking the coinbase address will be the address of bls public key + if utils.GetAddressFromBLSPubKeyBytes(member.BLSPublicKey[:]) == h.Coinbase() { + if committerKey, err = bls.BytesToBLSPublicKey(member.BLSPublicKey[:]); err != nil { + return nil, err + } + return &bls.PublicKeyWrapper{Object: committerKey, Bytes: member.BLSPublicKey}, nil + } + } else { + if member.EcdsaAddress == h.Coinbase() { + if committerKey, err = bls.BytesToBLSPublicKey(member.BLSPublicKey[:]); err != nil { + return nil, err + } + return &bls.PublicKeyWrapper{Object: committerKey, Bytes: member.BLSPublicKey}, nil + } + } + } + return nil, errors.Errorf( + "cannot find corresponding BLS Public Key coinbase %s", + h.Coinbase().Hex(), + ) +} + func (bc *BlockChainImpl) EnablePruneBeaconChainFeature() { bc.pruneBeaconChainEnable = true } diff --git a/core/blockchain_stub.go b/core/blockchain_stub.go index ccb1c98474..f9e9111eae 100644 --- a/core/blockchain_stub.go +++ b/core/blockchain_stub.go @@ -13,6 +13,7 @@ import ( "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" + "github.com/harmony-one/harmony/crypto/bls" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/tikv/redis_helper" @@ -403,6 +404,10 @@ func (a Stub) CommitOffChainData(batch rawdb.DatabaseWriter, block *types.Block, return 0, errors.Errorf("method CommitOffChainData not implemented for %s", a.Name) } +func (a Stub) GetLeaderPubKeyFromCoinbase(h *block.Header) (*bls.PublicKeyWrapper, error) { + return nil, errors.Errorf("method GetLeaderPubKeyFromCoinbase not implemented for %s", a.Name) +} + func (a Stub) IsTikvWriterMaster() bool { return false } From 4688002eab5fdcb7dadf056a3df1c39aa88b9b27 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 17 Oct 2022 15:08:07 +0700 Subject: [PATCH 016/420] activate epoch --- internal/params/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/params/config.go b/internal/params/config.go index 55e8126e80..9fa4a460a5 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -107,6 +107,8 @@ var ( ChainIdFixEpoch: big.NewInt(0), CrossShardXferPrecompileEpoch: big.NewInt(2), AllowlistEpoch: big.NewInt(2), + LeaderRotationEpoch: big.NewInt(290), + LeaderRotationBlocksCount: 64, FeeCollectEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. From 47d98824033ce106c370e5593ed6663d20d176b0 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 17 Oct 2022 17:09:04 +0700 Subject: [PATCH 017/420] comment activation --- internal/params/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/params/config.go b/internal/params/config.go index 9fa4a460a5..d7b6dd7e29 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -107,8 +107,8 @@ var ( ChainIdFixEpoch: big.NewInt(0), CrossShardXferPrecompileEpoch: big.NewInt(2), AllowlistEpoch: big.NewInt(2), - LeaderRotationEpoch: big.NewInt(290), - LeaderRotationBlocksCount: 64, + //LeaderRotationEpoch: big.NewInt(290), + LeaderRotationBlocksCount: 64, FeeCollectEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. From 8ad33f69a1839925104660bd0cccd546ab0b6ab8 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 18 Oct 2022 12:17:17 +0700 Subject: [PATCH 018/420] 295 epoch --- internal/params/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/params/config.go b/internal/params/config.go index d7b6dd7e29..fd920c3d83 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -107,8 +107,8 @@ var ( ChainIdFixEpoch: big.NewInt(0), CrossShardXferPrecompileEpoch: big.NewInt(2), AllowlistEpoch: big.NewInt(2), - //LeaderRotationEpoch: big.NewInt(290), - LeaderRotationBlocksCount: 64, + LeaderRotationEpoch: big.NewInt(295), + LeaderRotationBlocksCount: 64, FeeCollectEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. From 3e0d6d5dceebf6a4f60c8ce0ccba95853521a39c Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 8 Nov 2022 20:19:37 +0700 Subject: [PATCH 019/420] Fix failed tests. --- internal/params/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/params/config.go b/internal/params/config.go index fd920c3d83..5d968ddef7 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -305,8 +305,8 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } @@ -350,7 +350,7 @@ var ( // TODO place correct epoch number big.NewInt(1), // LeaderRotationEpoch 64, // LeaderRotationBlocksCount - big.NewInt(0), // FeeCollectEpoch + big.NewInt(0), // FeeCollectEpoch } // TestRules ... From 237b9715f8a529898c60d16220117c7fcaeeb0d3 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 9 Nov 2022 15:36:28 +0700 Subject: [PATCH 020/420] Fixed code review. --- consensus/consensus_v2.go | 2 +- hmy/hmy.go | 2 -- internal/params/config.go | 11 ++++++----- node/api.go | 6 ------ 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 5fe176c2f4..9d8702f63c 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -691,7 +691,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { if header == nil { return } - // Previous epoch, we should not change leader. + // Previous block was epoch block, we should not change leader. if header.Epoch().Uint64() != epoch.Uint64() { return } diff --git a/hmy/hmy.go b/hmy/hmy.go index 8089f04ba8..58e8d59edf 100644 --- a/hmy/hmy.go +++ b/hmy/hmy.go @@ -18,7 +18,6 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" - "github.com/harmony-one/harmony/multibls" commonRPC "github.com/harmony-one/harmony/rpc/common" "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" @@ -97,7 +96,6 @@ type NodeAPI interface { GetStakingTransactionsCount(address, txType string) (uint64, error) GetTraceResultByHash(hash common.Hash) (json.RawMessage, error) IsCurrentlyLeader() bool - GetPublicKeys() multibls.PublicKeys IsOutOfSync(shardID uint32) bool SyncStatus(shardID uint32) (bool, uint64, uint64) SyncPeers() map[string]int diff --git a/internal/params/config.go b/internal/params/config.go index 5d968ddef7..b69c8bee3e 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -226,6 +226,8 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, FeeCollectEpoch: EpochTBD, + LeaderRotationEpoch: EpochTBD, + LeaderRotationBlocksCount: 64, } // LocalnetChainConfig contains the chain parameters to run for local development. @@ -264,7 +266,7 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, FeeCollectEpoch: big.NewInt(5), - LeaderRotationEpoch: big.NewInt(4), + LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 5, } @@ -347,10 +349,9 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - // TODO place correct epoch number - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount - big.NewInt(0), // FeeCollectEpoch + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount + big.NewInt(0), // FeeCollectEpoch } // TestRules ... diff --git a/node/api.go b/node/api.go index debcb201f6..ceda968084 100644 --- a/node/api.go +++ b/node/api.go @@ -6,7 +6,6 @@ import ( "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy" "github.com/harmony-one/harmony/internal/tikv" - "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/rosetta" hmy_rpc "github.com/harmony-one/harmony/rpc" rpc_common "github.com/harmony-one/harmony/rpc/common" @@ -19,11 +18,6 @@ func (node *Node) IsCurrentlyLeader() bool { return node.Consensus.IsLeader() } -// GetPublicKeys exposes if node is currently the leader node -func (node *Node) GetPublicKeys() multibls.PublicKeys { - return node.Consensus.GetPrivateKeys().GetPublicKeys() -} - // PeerConnectivity .. func (node *Node) PeerConnectivity() (int, int, int) { return node.host.PeerConnectivity() From 478d0844a2579785831fc9a6c8690cb82c61a3f9 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 10 Nov 2022 01:42:00 +0700 Subject: [PATCH 021/420] Fix review "--port flag". --- internal/utils/singleton.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index 7aee4d86be..10101d7673 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -197,6 +197,7 @@ func updateZeroLogLevel(level int) { zeroLogger = &childLogger } +// GetPort is useful for debugging, returns `--port` flag provided to executable. func GetPort() int { ok := false for _, x := range os.Args { From 38f5437eda9a9c11c1d2b02df8ab917f3a25dd3b Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 10 Nov 2022 12:57:57 +0700 Subject: [PATCH 022/420] Fix review comments. --- consensus/consensus.go | 9 ++++++ consensus/consensus_v2.go | 66 +++++++++++++++++++-------------------- internal/params/config.go | 6 ++++ 3 files changed, 48 insertions(+), 33 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 1259576834..e0e096e306 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -180,12 +180,21 @@ func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper { defer consensus.pubKeyLock.Unlock() return consensus.LeaderPubKey } + +func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { + return consensus.LeaderPubKey +} + func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { consensus.pubKeyLock.Lock() consensus.LeaderPubKey = pub consensus.pubKeyLock.Unlock() } +func (consensus *Consensus) setLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { + consensus.LeaderPubKey = pub +} + func (consensus *Consensus) GetPrivateKeys() multibls.PrivateKeys { return consensus.priKey } diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 9d8702f63c..315fc332e7 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -680,46 +680,46 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess return nil } +// rotateLeader rotates the leader to the next leader in the committee. +// This function must be called with enabled leader rotation. func (consensus *Consensus) rotateLeader(epoch *big.Int) { - prev := consensus.GetLeaderPubKey() + prev := consensus.getLeaderPubKey() curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), consensus.Blockchain.Config().IsLeaderRotation(epoch), consensus.Blockchain.Config().LeaderRotationBlocksCount) - if consensus.Blockchain.Config().IsLeaderRotation(epoch) { - leader := consensus.GetLeaderPubKey() - for i := 0; i < consensus.Blockchain.Config().LeaderRotationBlocksCount; i++ { - header := consensus.Blockchain.GetHeaderByNumber(curNumber - uint64(i)) - if header == nil { - return - } - // Previous block was epoch block, we should not change leader. - if header.Epoch().Uint64() != epoch.Uint64() { - return - } - // Check if the same leader. - pub, err := consensus.Blockchain.GetLeaderPubKeyFromCoinbase(header) - if err != nil { - utils.Logger().Error().Err(err).Msg("Failed to get leader public key from coinbase") - return - } - if !pub.Object.IsEqual(leader.Object) { - // Another leader. - return - } + leader := consensus.getLeaderPubKey() + for i := 0; i < consensus.Blockchain.Config().LeaderRotationBlocksCount; i++ { + header := consensus.Blockchain.GetHeaderByNumber(curNumber - uint64(i)) + if header == nil { + return } - // Passed all checks, we can change leader. - wasFound, next := consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) - if !wasFound { - utils.Logger().Error().Msg("Failed to get next leader") + // Previous epoch, we should not change leader. + if header.Epoch().Uint64() != epoch.Uint64() { return - } else { - consensus.SetLeaderPubKey(next) } - if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { - // leader changed - go func() { - consensus.ReadySignal <- SyncProposal - }() + // Check if the same leader. + pub, err := consensus.Blockchain.GetLeaderPubKeyFromCoinbase(header) + if err != nil { + utils.Logger().Error().Err(err).Msg("Failed to get leader public key from coinbase") + return } + if !pub.Object.IsEqual(leader.Object) { + // Another leader. + return + } + } + // Passed all checks, we can change leader. + wasFound, next := consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) + if !wasFound { + utils.Logger().Error().Msg("Failed to get next leader") + return + } else { + consensus.setLeaderPubKey(next) + } + if consensus.isLeader() && !consensus.getLeaderPubKey().Object.IsEqual(prev.Object) { + // leader changed + go func() { + consensus.ReadySignal <- SyncProposal + }() } } diff --git a/internal/params/config.go b/internal/params/config.go index b69c8bee3e..3d40a7c6c9 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -70,6 +70,8 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(1323), // Around Wed 8 Feb 11:30PM UTC AllowlistEpoch: EpochTBD, FeeCollectEpoch: EpochTBD, + LeaderRotationEpoch: EpochTBD, + LeaderRotationBlocksCount: 64, } // TestnetChainConfig contains the chain parameters to run a node on the harmony test network. @@ -147,6 +149,8 @@ var ( SlotsLimitedEpoch: EpochTBD, // epoch to enable HIP-16 CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, + LeaderRotationEpoch: EpochTBD, + LeaderRotationBlocksCount: 64, FeeCollectEpoch: EpochTBD, } @@ -187,6 +191,8 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, FeeCollectEpoch: big.NewInt(574), + LeaderRotationEpoch: EpochTBD, + LeaderRotationBlocksCount: 64, } // StressnetChainConfig contains the chain parameters for the Stress test network. From 584d35a2f7b332575c1f5079ea3565f6d156dc87 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 10 Nov 2022 17:00:38 +0700 Subject: [PATCH 023/420] Returned locks in rotateLeader. --- consensus/consensus_v2.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 315fc332e7..ae57632481 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -683,10 +683,10 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess // rotateLeader rotates the leader to the next leader in the committee. // This function must be called with enabled leader rotation. func (consensus *Consensus) rotateLeader(epoch *big.Int) { - prev := consensus.getLeaderPubKey() + prev := consensus.GetLeaderPubKey() curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), consensus.Blockchain.Config().IsLeaderRotation(epoch), consensus.Blockchain.Config().LeaderRotationBlocksCount) - leader := consensus.getLeaderPubKey() + leader := consensus.GetLeaderPubKey() for i := 0; i < consensus.Blockchain.Config().LeaderRotationBlocksCount; i++ { header := consensus.Blockchain.GetHeaderByNumber(curNumber - uint64(i)) if header == nil { @@ -713,9 +713,9 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { utils.Logger().Error().Msg("Failed to get next leader") return } else { - consensus.setLeaderPubKey(next) + consensus.SetLeaderPubKey(next) } - if consensus.isLeader() && !consensus.getLeaderPubKey().Object.IsEqual(prev.Object) { + if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { // leader changed go func() { consensus.ReadySignal <- SyncProposal From a399586903c4f0932ccdc67faaa25d61c55f0dd4 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 17 Jan 2023 11:57:05 -0300 Subject: [PATCH 024/420] Rebased onto dev. --- consensus/consensus_v2.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index ae57632481..0b1ce28cf4 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -684,11 +684,12 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess // This function must be called with enabled leader rotation. func (consensus *Consensus) rotateLeader(epoch *big.Int) { prev := consensus.GetLeaderPubKey() - curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() - utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), consensus.Blockchain.Config().IsLeaderRotation(epoch), consensus.Blockchain.Config().LeaderRotationBlocksCount) + bc := consensus.Blockchain() + curNumber := bc.CurrentHeader().Number().Uint64() + utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), bc.Config().IsLeaderRotation(epoch), bc.Config().LeaderRotationBlocksCount) leader := consensus.GetLeaderPubKey() - for i := 0; i < consensus.Blockchain.Config().LeaderRotationBlocksCount; i++ { - header := consensus.Blockchain.GetHeaderByNumber(curNumber - uint64(i)) + for i := 0; i < bc.Config().LeaderRotationBlocksCount; i++ { + header := bc.GetHeaderByNumber(curNumber - uint64(i)) if header == nil { return } @@ -697,7 +698,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { return } // Check if the same leader. - pub, err := consensus.Blockchain.GetLeaderPubKeyFromCoinbase(header) + pub, err := bc.GetLeaderPubKeyFromCoinbase(header) if err != nil { utils.Logger().Error().Err(err).Msg("Failed to get leader public key from coinbase") return @@ -736,7 +737,7 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg } else { epoch = blk.Epoch() } - if consensus.Blockchain.Config().IsLeaderRotation(epoch) { + if consensus.Blockchain().Config().IsLeaderRotation(epoch) { consensus.rotateLeader(epoch) } From 3b1997e085746857c8949d3d8a0df99b43707894 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 17 Jan 2023 17:17:30 -0300 Subject: [PATCH 025/420] Commented golangci. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9d4982491b..514f8e2fb4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ install: - (cd $GOPATH/src/github.com/harmony-one/bls; make BLS_SWAP_G=1 -j4) - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1 - - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.41.1 +# - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.41.1 - make go-get - go install golang.org/x/tools/cmd/goimports@latest - go install github.com/harmony-ek/gencodec@latest From da831104293595fdeaa5f2c40124ba2b3bb931be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <“Gheis.Mohammadi@gmail.com”> Date: Tue, 27 Dec 2022 16:49:02 +0800 Subject: [PATCH 026/420] staged stream sync v1.0 --- .gitignore | 3 + api/service/manager.go | 3 + api/service/stagedstreamsync/adapter.go | 34 + api/service/stagedstreamsync/beacon_helper.go | 156 +++++ .../stagedstreamsync/block_by_hash_manager.go | 133 ++++ .../stagedstreamsync/block_hash_result.go | 75 +++ api/service/stagedstreamsync/block_manager.go | 172 +++++ api/service/stagedstreamsync/const.go | 81 +++ .../stagedstreamsync/default_stages.go | 87 +++ api/service/stagedstreamsync/downloader.go | 303 +++++++++ api/service/stagedstreamsync/downloaders.go | 96 +++ api/service/stagedstreamsync/errors.go | 56 ++ api/service/stagedstreamsync/helpers.go | 114 ++++ api/service/stagedstreamsync/metric.go | 98 +++ api/service/stagedstreamsync/service.go | 30 + .../stagedstreamsync/short_range_helper.go | 221 +++++++ api/service/stagedstreamsync/stage.go | 112 ++++ api/service/stagedstreamsync/stage_bodies.go | 420 ++++++++++++ api/service/stagedstreamsync/stage_epoch.go | 198 ++++++ api/service/stagedstreamsync/stage_finish.go | 114 ++++ api/service/stagedstreamsync/stage_heads.go | 157 +++++ .../stagedstreamsync/stage_short_range.go | 205 ++++++ api/service/stagedstreamsync/stage_state.go | 295 +++++++++ .../stagedstreamsync/staged_stream_sync.go | 597 ++++++++++++++++++ api/service/stagedstreamsync/stages.go | 71 +++ api/service/stagedstreamsync/syncing.go | 320 ++++++++++ api/service/stagedstreamsync/types.go | 287 +++++++++ api/service/stagedstreamsync/types_test.go | 266 ++++++++ cmd/harmony/main.go | 46 +- go.mod | 1 + go.sum | 2 + hmy/downloader/const.go | 3 +- hmy/downloader/metric.go | 14 +- hmy/downloader/shortrange.go | 2 +- node/node_syncing.go | 44 +- .../common/requestmanager/interface_test.go | 12 + .../common/streammanager/interface_test.go | 12 + .../common/streammanager/streammanager.go | 23 +- p2p/stream/protocols/sync/client.go | 30 + p2p/stream/protocols/sync/const.go | 3 + p2p/stream/protocols/sync/protocol.go | 62 +- p2p/stream/types/stream.go | 22 +- p2p/stream/types/utils.go | 37 +- 43 files changed, 4968 insertions(+), 49 deletions(-) create mode 100644 api/service/stagedstreamsync/adapter.go create mode 100644 api/service/stagedstreamsync/beacon_helper.go create mode 100644 api/service/stagedstreamsync/block_by_hash_manager.go create mode 100644 api/service/stagedstreamsync/block_hash_result.go create mode 100644 api/service/stagedstreamsync/block_manager.go create mode 100644 api/service/stagedstreamsync/const.go create mode 100644 api/service/stagedstreamsync/default_stages.go create mode 100644 api/service/stagedstreamsync/downloader.go create mode 100644 api/service/stagedstreamsync/downloaders.go create mode 100644 api/service/stagedstreamsync/errors.go create mode 100644 api/service/stagedstreamsync/helpers.go create mode 100644 api/service/stagedstreamsync/metric.go create mode 100644 api/service/stagedstreamsync/service.go create mode 100644 api/service/stagedstreamsync/short_range_helper.go create mode 100644 api/service/stagedstreamsync/stage.go create mode 100644 api/service/stagedstreamsync/stage_bodies.go create mode 100644 api/service/stagedstreamsync/stage_epoch.go create mode 100644 api/service/stagedstreamsync/stage_finish.go create mode 100644 api/service/stagedstreamsync/stage_heads.go create mode 100644 api/service/stagedstreamsync/stage_short_range.go create mode 100644 api/service/stagedstreamsync/stage_state.go create mode 100644 api/service/stagedstreamsync/staged_stream_sync.go create mode 100644 api/service/stagedstreamsync/stages.go create mode 100644 api/service/stagedstreamsync/syncing.go create mode 100644 api/service/stagedstreamsync/types.go create mode 100644 api/service/stagedstreamsync/types_test.go diff --git a/.gitignore b/.gitignore index 0e26bf59ae..6a3b3c4267 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,6 @@ explorer_storage_* # pprof profiles profiles/*.pb.gz + +# cache db +cache_*_db \ No newline at end of file diff --git a/api/service/manager.go b/api/service/manager.go index 57ca15d607..d879b6a6cd 100644 --- a/api/service/manager.go +++ b/api/service/manager.go @@ -23,6 +23,7 @@ const ( Prometheus Synchronize CrosslinkSending + StagedStreamSync ) func (t Type) String() string { @@ -45,6 +46,8 @@ func (t Type) String() string { return "Synchronize" case CrosslinkSending: return "CrosslinkSending" + case StagedStreamSync: + return "StagedStreamSync" default: return "Unknown" } diff --git a/api/service/stagedstreamsync/adapter.go b/api/service/stagedstreamsync/adapter.go new file mode 100644 index 0000000000..ae7632889c --- /dev/null +++ b/api/service/stagedstreamsync/adapter.go @@ -0,0 +1,34 @@ +package stagedstreamsync + +import ( + "context" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/event" + "github.com/harmony-one/harmony/consensus/engine" + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/p2p/stream/common/streammanager" + syncproto "github.com/harmony-one/harmony/p2p/stream/protocols/sync" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" +) + +type syncProtocol interface { + GetCurrentBlockNumber(ctx context.Context, opts ...syncproto.Option) (uint64, sttypes.StreamID, error) + GetBlocksByNumber(ctx context.Context, bns []uint64, opts ...syncproto.Option) ([]*types.Block, sttypes.StreamID, error) + GetRawBlocksByNumber(ctx context.Context, bns []uint64, opts ...syncproto.Option) ([][]byte, [][]byte, sttypes.StreamID, error) + GetBlockHashes(ctx context.Context, bns []uint64, opts ...syncproto.Option) ([]common.Hash, sttypes.StreamID, error) + GetBlocksByHashes(ctx context.Context, hs []common.Hash, opts ...syncproto.Option) ([]*types.Block, sttypes.StreamID, error) + + RemoveStream(stID sttypes.StreamID) // If a stream delivers invalid data, remove the stream + StreamFailed(stID sttypes.StreamID, reason string) + SubscribeAddStreamEvent(ch chan<- streammanager.EvtStreamAdded) event.Subscription + NumStreams() int +} + +type blockChain interface { + engine.ChainReader + Engine() engine.Engine + + InsertChain(chain types.Blocks, verifyHeaders bool) (int, error) + WriteCommitSig(blockNum uint64, lastCommits []byte) error +} diff --git a/api/service/stagedstreamsync/beacon_helper.go b/api/service/stagedstreamsync/beacon_helper.go new file mode 100644 index 0000000000..4a77020160 --- /dev/null +++ b/api/service/stagedstreamsync/beacon_helper.go @@ -0,0 +1,156 @@ +package stagedstreamsync + +import ( + "time" + + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/internal/utils" + "github.com/rs/zerolog" +) + +// lastMileCache keeps the last 50 number blocks in memory cache +const lastMileCap = 50 + +type ( + // beaconHelper is the helper for the beacon downloader. The beaconHelper is only started + // when node is running on side chain, listening to beacon client pub-sub message and + // insert the latest blocks to the beacon chain. + beaconHelper struct { + bc blockChain + blockC <-chan *types.Block + // TODO: refactor this hook to consensus module. We'd better put it in + // consensus module under a subscription. + insertHook func() + + lastMileCache *blocksByNumber + insertC chan insertTask + closeC chan struct{} + logger zerolog.Logger + } + + insertTask struct { + doneC chan struct{} + } +) + +func newBeaconHelper(bc blockChain, blockC <-chan *types.Block, insertHook func()) *beaconHelper { + return &beaconHelper{ + bc: bc, + blockC: blockC, + insertHook: insertHook, + lastMileCache: newBlocksByNumber(lastMileCap), + insertC: make(chan insertTask, 1), + closeC: make(chan struct{}), + logger: utils.Logger().With(). + Str("module", "downloader"). + Str("sub-module", "beacon helper"). + Logger(), + } +} + +func (bh *beaconHelper) start() { + go bh.loop() +} + +func (bh *beaconHelper) close() { + close(bh.closeC) +} + +func (bh *beaconHelper) loop() { + t := time.NewTicker(10 * time.Second) + defer t.Stop() + for { + select { + case <-t.C: + bh.insertAsync() + + case b, ok := <-bh.blockC: + if !ok { + return // blockC closed. Node exited + } + if b == nil { + continue + } + bh.lastMileCache.push(b) + bh.insertAsync() + + case it := <-bh.insertC: + inserted, bn, err := bh.insertLastMileBlocks() + numBlocksInsertedBeaconHelperCounter.Add(float64(inserted)) + if err != nil { + bh.logger.Error().Err(err).Msg(WrapStagedSyncMsg("insert last mile blocks error")) + continue + } + bh.logger.Info().Int("inserted", inserted). + Uint64("end height", bn). + Uint32("shard", bh.bc.ShardID()). + Msg(WrapStagedSyncMsg("insert last mile blocks")) + + close(it.doneC) + + case <-bh.closeC: + return + } + } +} + +// insertAsync triggers the insert last mile without blocking +func (bh *beaconHelper) insertAsync() { + select { + case bh.insertC <- insertTask{ + doneC: make(chan struct{}), + }: + default: + } +} + +// insertSync triggers the insert last mile while blocking +func (bh *beaconHelper) insertSync() { + task := insertTask{ + doneC: make(chan struct{}), + } + bh.insertC <- task + <-task.doneC +} + +func (bh *beaconHelper) insertLastMileBlocks() (inserted int, bn uint64, err error) { + bn = bh.bc.CurrentBlock().NumberU64() + 1 + for { + b := bh.getNextBlock(bn) + if b == nil { + bn-- + return + } + // TODO: Instruct the beacon helper to verify signatures. This may require some forks + // in pub-sub message (add commit sigs in node.block.sync messages) + if _, err = bh.bc.InsertChain(types.Blocks{b}, true); err != nil { + bn-- + return + } + bh.logger.Info().Uint64("number", b.NumberU64()).Msg(WrapStagedSyncMsg("Inserted block from beacon pub-sub")) + + if bh.insertHook != nil { + bh.insertHook() + } + inserted++ + bn++ + } +} + +func (bh *beaconHelper) getNextBlock(expBN uint64) *types.Block { + for bh.lastMileCache.len() > 0 { + b := bh.lastMileCache.pop() + if b == nil { + return nil + } + if b.NumberU64() < expBN { + continue + } + if b.NumberU64() > expBN { + bh.lastMileCache.push(b) + return nil + } + return b + } + return nil +} diff --git a/api/service/stagedstreamsync/block_by_hash_manager.go b/api/service/stagedstreamsync/block_by_hash_manager.go new file mode 100644 index 0000000000..86abc22ed7 --- /dev/null +++ b/api/service/stagedstreamsync/block_by_hash_manager.go @@ -0,0 +1,133 @@ +package stagedstreamsync + +import ( + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/harmony/core/types" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/pkg/errors" +) + +type getBlocksByHashManager struct { + hashes []common.Hash + pendings map[common.Hash]struct{} + results map[common.Hash]blockResult + whitelist []sttypes.StreamID + + lock sync.Mutex +} + +func newGetBlocksByHashManager(hashes []common.Hash, whitelist []sttypes.StreamID) *getBlocksByHashManager { + return &getBlocksByHashManager{ + hashes: hashes, + pendings: make(map[common.Hash]struct{}), + results: make(map[common.Hash]blockResult), + whitelist: whitelist, + } +} + +func (m *getBlocksByHashManager) getNextHashes() ([]common.Hash, []sttypes.StreamID, error) { + m.lock.Lock() + defer m.lock.Unlock() + + num := m.numBlocksPerRequest() + hashes := make([]common.Hash, 0, num) + if len(m.whitelist) == 0 { + return nil, nil, errors.New("empty white list") + } + + for _, hash := range m.hashes { + if len(hashes) == num { + break + } + _, ok1 := m.pendings[hash] + _, ok2 := m.results[hash] + if !ok1 && !ok2 { + hashes = append(hashes, hash) + } + } + sts := make([]sttypes.StreamID, len(m.whitelist)) + copy(sts, m.whitelist) + return hashes, sts, nil +} + +func (m *getBlocksByHashManager) numBlocksPerRequest() int { + val := divideCeil(len(m.hashes), len(m.whitelist)) + if val < BlockByHashesLowerCap { + val = BlockByHashesLowerCap + } + if val > BlockByHashesUpperCap { + val = BlockByHashesUpperCap + } + return val +} + +func (m *getBlocksByHashManager) numRequests() int { + return divideCeil(len(m.hashes), m.numBlocksPerRequest()) +} + +func (m *getBlocksByHashManager) addResult(hashes []common.Hash, blocks []*types.Block, stid sttypes.StreamID) { + m.lock.Lock() + defer m.lock.Unlock() + + for i, hash := range hashes { + block := blocks[i] + delete(m.pendings, hash) + m.results[hash] = blockResult{ + block: block, + stid: stid, + } + } +} + +func (m *getBlocksByHashManager) handleResultError(hashes []common.Hash, stid sttypes.StreamID) { + m.lock.Lock() + defer m.lock.Unlock() + + m.removeStreamID(stid) + + for _, hash := range hashes { + delete(m.pendings, hash) + } +} + +func (m *getBlocksByHashManager) getResults() ([]*types.Block, []sttypes.StreamID, error) { + m.lock.Lock() + defer m.lock.Unlock() + + blocks := make([]*types.Block, 0, len(m.hashes)) + stids := make([]sttypes.StreamID, 0, len(m.hashes)) + for _, hash := range m.hashes { + if m.results[hash].block == nil { + return nil, nil, errors.New("SANITY: nil block found") + } + blocks = append(blocks, m.results[hash].block) + stids = append(stids, m.results[hash].stid) + } + return blocks, stids, nil +} + +func (m *getBlocksByHashManager) isDone() bool { + m.lock.Lock() + defer m.lock.Unlock() + + return len(m.results) == len(m.hashes) +} + +func (m *getBlocksByHashManager) removeStreamID(target sttypes.StreamID) { + // O(n^2) complexity. But considering the whitelist size is small, should not + // have performance issue. +loop: + for i, stid := range m.whitelist { + if stid == target { + if i == len(m.whitelist) { + m.whitelist = m.whitelist[:i] + } else { + m.whitelist = append(m.whitelist[:i], m.whitelist[i+1:]...) + } + goto loop + } + } + return +} diff --git a/api/service/stagedstreamsync/block_hash_result.go b/api/service/stagedstreamsync/block_hash_result.go new file mode 100644 index 0000000000..0bae60507d --- /dev/null +++ b/api/service/stagedstreamsync/block_hash_result.go @@ -0,0 +1,75 @@ +package stagedstreamsync + +import ( + "sync" + + "github.com/ethereum/go-ethereum/common" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" +) + +type ( + blockHashResults struct { + bns []uint64 + results []map[sttypes.StreamID]common.Hash + + lock sync.Mutex + } +) + +func newBlockHashResults(bns []uint64) *blockHashResults { + results := make([]map[sttypes.StreamID]common.Hash, 0, len(bns)) + for range bns { + results = append(results, make(map[sttypes.StreamID]common.Hash)) + } + return &blockHashResults{ + bns: bns, + results: results, + } +} + +func (res *blockHashResults) addResult(hashes []common.Hash, stid sttypes.StreamID) { + res.lock.Lock() + defer res.lock.Unlock() + + for i, h := range hashes { + if h == emptyHash { + return // nil block hash reached + } + res.results[i][stid] = h + } + return +} + +func (res *blockHashResults) computeLongestHashChain() ([]common.Hash, []sttypes.StreamID) { + var ( + whitelist map[sttypes.StreamID]struct{} + hashChain []common.Hash + ) + for _, result := range res.results { + hash, nextWl := countHashMaxVote(result, whitelist) + if hash == emptyHash { + break + } + hashChain = append(hashChain, hash) + whitelist = nextWl + } + + sts := make([]sttypes.StreamID, 0, len(whitelist)) + for st := range whitelist { + sts = append(sts, st) + } + return hashChain, sts +} + +func (res *blockHashResults) numBlocksWithResults() int { + res.lock.Lock() + defer res.lock.Unlock() + + cnt := 0 + for _, result := range res.results { + if len(result) != 0 { + cnt++ + } + } + return cnt +} diff --git a/api/service/stagedstreamsync/block_manager.go b/api/service/stagedstreamsync/block_manager.go new file mode 100644 index 0000000000..df30ab5e36 --- /dev/null +++ b/api/service/stagedstreamsync/block_manager.go @@ -0,0 +1,172 @@ +package stagedstreamsync + +import ( + "sync" + + sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/rs/zerolog" +) + +type BlockDownloadDetails struct { + loopID int + streamID sttypes.StreamID +} + +// blockDownloadManager is the helper structure for get blocks request management +type blockDownloadManager struct { + chain blockChain + tx kv.RwTx + + targetBN uint64 + requesting map[uint64]struct{} // block numbers that have been assigned to workers but not received + processing map[uint64]struct{} // block numbers received requests but not inserted + retries *prioritizedNumbers // requests where error happens + rq *resultQueue // result queue wait to be inserted into blockchain + bdd map[uint64]BlockDownloadDetails // details about how this block was downloaded + + logger zerolog.Logger + lock sync.Mutex +} + +func newBlockDownloadManager(tx kv.RwTx, chain blockChain, targetBN uint64, logger zerolog.Logger) *blockDownloadManager { + return &blockDownloadManager{ + chain: chain, + tx: tx, + targetBN: targetBN, + requesting: make(map[uint64]struct{}), + processing: make(map[uint64]struct{}), + retries: newPrioritizedNumbers(), + rq: newResultQueue(), + bdd: make(map[uint64]BlockDownloadDetails), + logger: logger, + } +} + +// GetNextBatch get the next block numbers batch +func (gbm *blockDownloadManager) GetNextBatch() []uint64 { + gbm.lock.Lock() + defer gbm.lock.Unlock() + + cap := BlocksPerRequest + + bns := gbm.getBatchFromRetries(cap) + if len(bns) > 0 { + cap -= len(bns) + gbm.addBatchToRequesting(bns) + } + + if gbm.availableForMoreTasks() { + addBNs := gbm.getBatchFromUnprocessed(cap) + gbm.addBatchToRequesting(addBNs) + bns = append(bns, addBNs...) + } + + return bns +} + +// HandleRequestError handles the error result +func (gbm *blockDownloadManager) HandleRequestError(bns []uint64, err error, streamID sttypes.StreamID) { + gbm.lock.Lock() + defer gbm.lock.Unlock() + + // add requested block numbers to retries + for _, bn := range bns { + delete(gbm.requesting, bn) + gbm.retries.push(bn) + } +} + +// HandleRequestResult handles get blocks result +func (gbm *blockDownloadManager) HandleRequestResult(bns []uint64, blockBytes [][]byte, sigBytes [][]byte, loopID int, streamID sttypes.StreamID) error { + gbm.lock.Lock() + defer gbm.lock.Unlock() + + for i, bn := range bns { + delete(gbm.requesting, bn) + if len(blockBytes[i]) <= 1 { + gbm.retries.push(bn) + } else { + gbm.processing[bn] = struct{}{} + gbm.bdd[bn] = BlockDownloadDetails{ + loopID: loopID, + streamID: streamID, + } + } + } + return nil +} + +// SetDownloadDetails sets the download details for a batch of blocks +func (gbm *blockDownloadManager) SetDownloadDetails(bns []uint64, loopID int, streamID sttypes.StreamID) error { + gbm.lock.Lock() + defer gbm.lock.Unlock() + + for _, bn := range bns { + gbm.bdd[bn] = BlockDownloadDetails{ + loopID: loopID, + streamID: streamID, + } + } + return nil +} + +// GetDownloadDetails returns the download details for a block +func (gbm *blockDownloadManager) GetDownloadDetails(blockNumber uint64) (loopID int, streamID sttypes.StreamID) { + gbm.lock.Lock() + defer gbm.lock.Unlock() + + return gbm.bdd[blockNumber].loopID, gbm.bdd[blockNumber].streamID +} + +// getBatchFromRetries get the block number batch to be requested from retries. +func (gbm *blockDownloadManager) getBatchFromRetries(cap int) []uint64 { + var ( + requestBNs []uint64 + curHeight = gbm.chain.CurrentBlock().NumberU64() + ) + for cnt := 0; cnt < cap; cnt++ { + bn := gbm.retries.pop() + if bn == 0 { + break // no more retries + } + if bn <= curHeight { + continue + } + requestBNs = append(requestBNs, bn) + } + return requestBNs +} + +// getBatchFromUnprocessed returns a batch of block numbers to be requested from unprocessed. +func (gbm *blockDownloadManager) getBatchFromUnprocessed(cap int) []uint64 { + var ( + requestBNs []uint64 + curHeight = gbm.chain.CurrentBlock().NumberU64() + ) + bn := curHeight + 1 + // TODO: this algorithm can be potentially optimized. + for cnt := 0; cnt < cap && bn <= gbm.targetBN; cnt++ { + for bn <= gbm.targetBN { + _, ok1 := gbm.requesting[bn] + _, ok2 := gbm.processing[bn] + if !ok1 && !ok2 { + requestBNs = append(requestBNs, bn) + bn++ + break + } + bn++ + } + } + return requestBNs +} + +func (gbm *blockDownloadManager) availableForMoreTasks() bool { + return gbm.rq.results.Len() < SoftQueueCap +} + +func (gbm *blockDownloadManager) addBatchToRequesting(bns []uint64) { + for _, bn := range bns { + gbm.requesting[bn] = struct{}{} + } +} diff --git a/api/service/stagedstreamsync/const.go b/api/service/stagedstreamsync/const.go new file mode 100644 index 0000000000..721b44d9f5 --- /dev/null +++ b/api/service/stagedstreamsync/const.go @@ -0,0 +1,81 @@ +package stagedstreamsync + +import ( + "time" + + "github.com/harmony-one/harmony/core/types" + nodeconfig "github.com/harmony-one/harmony/internal/configs/node" +) + +const ( + BlocksPerRequest int = 10 // number of blocks for each request + BlocksPerInsertion int = 50 // number of blocks for each insert batch + BlockHashesPerRequest int = 20 // number of get block hashes for short range sync + BlockByHashesUpperCap int = 10 // number of get blocks by hashes upper cap + BlockByHashesLowerCap int = 3 // number of get blocks by hashes lower cap + + LastMileBlocksThreshold int = 10 + + // SoftQueueCap is the soft cap of size in resultQueue. When the queue size is larger than this limit, + // no more request will be assigned to workers to wait for InsertChain to finish. + SoftQueueCap int = 100 + + // DefaultConcurrency is the default settings for concurrency + DefaultConcurrency = 4 + + // ShortRangeTimeout is the timeout for each short range sync, which allow short range sync + // to restart automatically when stuck in `getBlockHashes` + ShortRangeTimeout = 1 * time.Minute +) + +type ( + // Config is the downloader config + Config struct { + // Only run stream sync protocol as a server. + // TODO: remove this when stream sync is fully up. + ServerOnly bool + + // parameters + Network nodeconfig.NetworkType + Concurrency int // Number of concurrent sync requests + MinStreams int // Minimum number of streams to do sync + InitStreams int // Number of streams requirement for initial bootstrap + + // stream manager config + SmSoftLowCap int + SmHardLowCap int + SmHiCap int + SmDiscBatch int + + // config for beacon config + BHConfig *BeaconHelperConfig + + // log the stage progress + LogProgress bool + } + + // BeaconHelperConfig is the extra config used for beaconHelper which uses + // pub-sub block message to do sync. + BeaconHelperConfig struct { + BlockC <-chan *types.Block + InsertHook func() + } +) + +func (c *Config) fixValues() { + if c.Concurrency == 0 { + c.Concurrency = DefaultConcurrency + } + if c.Concurrency > c.MinStreams { + c.MinStreams = c.Concurrency + } + if c.MinStreams > c.InitStreams { + c.InitStreams = c.MinStreams + } + if c.MinStreams > c.SmSoftLowCap { + c.SmSoftLowCap = c.MinStreams + } + if c.MinStreams > c.SmHardLowCap { + c.SmHardLowCap = c.MinStreams + } +} diff --git a/api/service/stagedstreamsync/default_stages.go b/api/service/stagedstreamsync/default_stages.go new file mode 100644 index 0000000000..6e4808738f --- /dev/null +++ b/api/service/stagedstreamsync/default_stages.go @@ -0,0 +1,87 @@ +package stagedstreamsync + +import ( + "context" +) + +type ForwardOrder []SyncStageID +type RevertOrder []SyncStageID +type CleanUpOrder []SyncStageID + +var DefaultForwardOrder = ForwardOrder{ + Heads, + SyncEpoch, + ShortRange, + BlockBodies, + // Stages below don't use Internet + States, + Finish, +} + +var DefaultRevertOrder = RevertOrder{ + Finish, + States, + BlockBodies, + ShortRange, + SyncEpoch, + Heads, +} + +var DefaultCleanUpOrder = CleanUpOrder{ + Finish, + States, + BlockBodies, + ShortRange, + SyncEpoch, + Heads, +} + +func DefaultStages(ctx context.Context, + headsCfg StageHeadsCfg, + seCfg StageEpochCfg, + srCfg StageShortRangeCfg, + bodiesCfg StageBodiesCfg, + statesCfg StageStatesCfg, + finishCfg StageFinishCfg, +) []*Stage { + + handlerStageHeads := NewStageHeads(headsCfg) + handlerStageShortRange := NewStageShortRange(srCfg) + handlerStageEpochSync := NewStageEpoch(seCfg) + handlerStageBodies := NewStageBodies(bodiesCfg) + handlerStageStates := NewStageStates(statesCfg) + handlerStageFinish := NewStageFinish(finishCfg) + + return []*Stage{ + { + ID: Heads, + Description: "Retrieve Chain Heads", + Handler: handlerStageHeads, + }, + { + ID: SyncEpoch, + Description: "Sync only Last Block of Epoch", + Handler: handlerStageEpochSync, + }, + { + ID: ShortRange, + Description: "Short Range Sync", + Handler: handlerStageShortRange, + }, + { + ID: BlockBodies, + Description: "Retrieve Block Bodies", + Handler: handlerStageBodies, + }, + { + ID: States, + Description: "Update Blockchain State", + Handler: handlerStageStates, + }, + { + ID: Finish, + Description: "Finalize Changes", + Handler: handlerStageFinish, + }, + } +} diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go new file mode 100644 index 0000000000..9fdd2f78f4 --- /dev/null +++ b/api/service/stagedstreamsync/downloader.go @@ -0,0 +1,303 @@ +package stagedstreamsync + +import ( + "context" + "fmt" + "time" + + "github.com/ethereum/go-ethereum/event" + "github.com/pkg/errors" + "github.com/rs/zerolog" + + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/crypto/bls" + "github.com/harmony-one/harmony/internal/chain" + nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/p2p" + "github.com/harmony-one/harmony/p2p/stream/common/streammanager" + "github.com/harmony-one/harmony/p2p/stream/protocols/sync" + "github.com/harmony-one/harmony/shard" +) + +type ( + // Downloader is responsible for sync task of one shard + Downloader struct { + bc blockChain + syncProtocol syncProtocol + bh *beaconHelper + stagedSyncInstance *StagedStreamSync + + downloadC chan struct{} + closeC chan struct{} + ctx context.Context + cancel func() + + config Config + logger zerolog.Logger + } +) + +// NewDownloader creates a new downloader +func NewDownloader(host p2p.Host, bc core.BlockChain, config Config) *Downloader { + config.fixValues() + + sp := sync.NewProtocol(sync.Config{ + Chain: bc, + Host: host.GetP2PHost(), + Discovery: host.GetDiscovery(), + ShardID: nodeconfig.ShardID(bc.ShardID()), + Network: config.Network, + + SmSoftLowCap: config.SmSoftLowCap, + SmHardLowCap: config.SmHardLowCap, + SmHiCap: config.SmHiCap, + DiscBatch: config.SmDiscBatch, + }) + + host.AddStreamProtocol(sp) + + var bh *beaconHelper + if config.BHConfig != nil && bc.ShardID() == shard.BeaconChainShardID { + bh = newBeaconHelper(bc, config.BHConfig.BlockC, config.BHConfig.InsertHook) + } + + logger := utils.Logger().With().Str("module", "StagedStreamSync").Uint32("ShardID", bc.ShardID()).Logger() + + ctx, cancel := context.WithCancel(context.Background()) + + //TODO: use mem db should be in config file + stagedSyncInstance, err := CreateStagedSync(ctx, bc, false, sp, config, logger, config.LogProgress) + if err != nil { + return nil + } + + return &Downloader{ + bc: bc, + syncProtocol: sp, + bh: bh, + stagedSyncInstance: stagedSyncInstance, + + downloadC: make(chan struct{}), + closeC: make(chan struct{}), + ctx: ctx, + cancel: cancel, + + config: config, + logger: logger, + } +} + +// Start starts the downloader +func (d *Downloader) Start() { + go func() { + d.waitForBootFinish() + fmt.Printf("boot completed for shard %d, %d streams are connected\n", d.bc.ShardID(), d.syncProtocol.NumStreams()) + d.loop() + }() + + if d.bh != nil { + d.bh.start() + } +} + +// Close closes the downloader +func (d *Downloader) Close() { + close(d.closeC) + d.cancel() + + if d.bh != nil { + d.bh.close() + } +} + +// DownloadAsync triggers the download async. +func (d *Downloader) DownloadAsync() { + select { + case d.downloadC <- struct{}{}: + consensusTriggeredDownloadCounterVec.With(d.promLabels()).Inc() + + case <-time.After(100 * time.Millisecond): + } +} + +// NumPeers returns the number of peers connected of a specific shard. +func (d *Downloader) NumPeers() int { + return d.syncProtocol.NumStreams() +} + +// IsSyncing returns the current sync status +func (d *Downloader) SyncStatus() (bool, uint64, uint64) { + syncing, target := d.stagedSyncInstance.status.get() + if !syncing { + target = d.bc.CurrentBlock().NumberU64() + } + return syncing, target, 0 +} + +// SubscribeDownloadStarted subscribes download started +func (d *Downloader) SubscribeDownloadStarted(ch chan struct{}) event.Subscription { + d.stagedSyncInstance.evtDownloadStartedSubscribed = true + return d.stagedSyncInstance.evtDownloadStarted.Subscribe(ch) +} + +// SubscribeDownloadFinished subscribes the download finished +func (d *Downloader) SubscribeDownloadFinished(ch chan struct{}) event.Subscription { + d.stagedSyncInstance.evtDownloadFinishedSubscribed = true + return d.stagedSyncInstance.evtDownloadFinished.Subscribe(ch) +} + +// waitForBootFinish waits for stream manager to finish the initial discovery and have +// enough peers to start downloader +func (d *Downloader) waitForBootFinish() { + evtCh := make(chan streammanager.EvtStreamAdded, 1) + sub := d.syncProtocol.SubscribeAddStreamEvent(evtCh) + defer sub.Unsubscribe() + + checkCh := make(chan struct{}, 1) + trigger := func() { + select { + case checkCh <- struct{}{}: + default: + } + } + trigger() + + t := time.NewTicker(10 * time.Second) + defer t.Stop() + for { + select { + case <-t.C: + trigger() + + case <-evtCh: + trigger() + + case <-checkCh: + if d.syncProtocol.NumStreams() >= d.config.InitStreams { + return + } + case <-d.closeC: + return + } + } +} + +func (d *Downloader) loop() { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + initSync := d.bc.ShardID() != shard.BeaconChainShardID + trigger := func() { + select { + case d.downloadC <- struct{}{}: + case <-time.After(100 * time.Millisecond): + } + } + go trigger() + + for { + select { + case <-ticker.C: + go trigger() + + case <-d.downloadC: + addedBN, err := d.stagedSyncInstance.doSync(d.ctx, initSync) + if err != nil { + //TODO: if there is a bad block which can't be resolved + if d.stagedSyncInstance.invalidBlock.Active { + numTriedStreams := len(d.stagedSyncInstance.invalidBlock.StreamID) + // if many streams couldn't solve it, then that's an unresolvable bad block + if numTriedStreams >= d.config.InitStreams { + if !d.stagedSyncInstance.invalidBlock.IsLogged { + fmt.Println("unresolvable bad block:", d.stagedSyncInstance.invalidBlock.Number) + d.stagedSyncInstance.invalidBlock.IsLogged = true + } + //TODO: if we don't have any new or untried stream in the list, sleep or panic + } + } + + // If error happens, sleep 5 seconds and retry + d.logger.Error(). + Err(err). + Bool("initSync", initSync). + Msg(WrapStagedSyncMsg("sync loop failed")) + go func() { + time.Sleep(5 * time.Second) + trigger() + }() + time.Sleep(1 * time.Second) + continue + } + d.logger.Info().Int("block added", addedBN). + Uint64("current height", d.bc.CurrentBlock().NumberU64()). + Bool("initSync", initSync). + Uint32("shard", d.bc.ShardID()). + Msg(WrapStagedSyncMsg("sync finished")) + + if addedBN != 0 { + // If block number has been changed, trigger another sync + // and try to add last mile from pub-sub (blocking) + go trigger() + if d.bh != nil { + d.bh.insertSync() + } + } + d.stagedSyncInstance.initSync = false + initSync = false + + case <-d.closeC: + return + } + } +} + +var emptySigVerifyErr *sigVerifyErr + +type sigVerifyErr struct { + err error +} + +func (e *sigVerifyErr) Error() string { + return fmt.Sprintf("[VerifyHeaderSignature] %v", e.err.Error()) +} + +func verifyAndInsertBlocks(bc blockChain, blocks types.Blocks) (int, error) { + for i, block := range blocks { + if err := verifyAndInsertBlock(bc, block, blocks[i+1:]...); err != nil { + return i, err + } + } + return len(blocks), nil +} + +func verifyAndInsertBlock(bc blockChain, block *types.Block, nextBlocks ...*types.Block) error { + var ( + sigBytes bls.SerializedSignature + bitmap []byte + err error + ) + if len(nextBlocks) > 0 { + // get commit sig from the next block + next := nextBlocks[0] + sigBytes = next.Header().LastCommitSignature() + bitmap = next.Header().LastCommitBitmap() + } else { + // get commit sig from current block + sigBytes, bitmap, err = chain.ParseCommitSigAndBitmap(block.GetCurrentCommitSig()) + if err != nil { + return errors.Wrap(err, "parse commitSigAndBitmap") + } + } + + if err := bc.Engine().VerifyHeaderSignature(bc, block.Header(), sigBytes, bitmap); err != nil { + return &sigVerifyErr{err} + } + if err := bc.Engine().VerifyHeader(bc, block.Header(), true); err != nil { + return errors.Wrap(err, "[VerifyHeader]") + } + if _, err := bc.InsertChain(types.Blocks{block}, false); err != nil { + return errors.Wrap(err, "[InsertChain]") + } + return nil +} diff --git a/api/service/stagedstreamsync/downloaders.go b/api/service/stagedstreamsync/downloaders.go new file mode 100644 index 0000000000..7b5881f100 --- /dev/null +++ b/api/service/stagedstreamsync/downloaders.go @@ -0,0 +1,96 @@ +package stagedstreamsync + +import ( + "github.com/harmony-one/abool" + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/p2p" +) + +// Downloaders is the set of downloaders +type Downloaders struct { + ds map[uint32]*Downloader + active *abool.AtomicBool + + config Config +} + +// NewDownloaders creates Downloaders for sync of multiple blockchains +func NewDownloaders(host p2p.Host, bcs []core.BlockChain, config Config) *Downloaders { + ds := make(map[uint32]*Downloader) + + for _, bc := range bcs { + if bc == nil { + continue + } + if _, ok := ds[bc.ShardID()]; ok { + continue + } + ds[bc.ShardID()] = NewDownloader(host, bc, config) + } + return &Downloaders{ + ds: ds, + active: abool.New(), + config: config, + } +} + +// Start starts the downloaders +func (ds *Downloaders) Start() { + if ds.config.ServerOnly { + // Run in server only mode. Do not start downloaders. + return + } + ds.active.Set() + for _, d := range ds.ds { + d.Start() + } +} + +// Close closes the downloaders +func (ds *Downloaders) Close() { + if ds.config.ServerOnly { + // Run in server only mode. Downloaders not started. + return + } + ds.active.UnSet() + for _, d := range ds.ds { + d.Close() + } +} + +// DownloadAsync triggers a download +func (ds *Downloaders) DownloadAsync(shardID uint32) { + d, ok := ds.ds[shardID] + if !ok && d != nil { + d.DownloadAsync() + } +} + +// GetShardDownloader returns the downloader with the given shard ID +func (ds *Downloaders) GetShardDownloader(shardID uint32) *Downloader { + return ds.ds[shardID] +} + +// NumPeers returns the connected peers for each shard +func (ds *Downloaders) NumPeers() map[uint32]int { + res := make(map[uint32]int) + + for sid, d := range ds.ds { + res[sid] = d.NumPeers() + } + return res +} + +// SyncStatus returns whether the given shard is doing syncing task and the target block number +func (ds *Downloaders) SyncStatus(shardID uint32) (bool, uint64, uint64) { + d, ok := ds.ds[shardID] + if !ok { + return false, 0, 0 + } + return d.SyncStatus() +} + +// IsActive returns whether the downloader is active +func (ds *Downloaders) IsActive() bool { + return ds.active.IsSet() +} diff --git a/api/service/stagedstreamsync/errors.go b/api/service/stagedstreamsync/errors.go new file mode 100644 index 0000000000..e4828003d2 --- /dev/null +++ b/api/service/stagedstreamsync/errors.go @@ -0,0 +1,56 @@ +package stagedstreamsync + +import ( + "fmt" +) + +// Errors ... +var ( + ErrRegistrationFail = WrapStagedSyncError("registration failed") + ErrGetBlock = WrapStagedSyncError("get block failed") + ErrGetBlockHash = WrapStagedSyncError("get block hash failed") + ErrGetConsensusHashes = WrapStagedSyncError("get consensus hashes failed") + ErrGenStateSyncTaskQueue = WrapStagedSyncError("generate state sync task queue failed") + ErrDownloadBlocks = WrapStagedSyncError("get download blocks failed") + ErrUpdateBlockAndStatus = WrapStagedSyncError("update block and status failed") + ErrGenerateNewState = WrapStagedSyncError("get generate new state failed") + ErrFetchBlockHashProgressFail = WrapStagedSyncError("fetch cache progress for block hashes stage failed") + ErrFetchCachedBlockHashFail = WrapStagedSyncError("fetch cached block hashes failed") + ErrNotEnoughBlockHashes = WrapStagedSyncError("peers haven't sent all requested block hashes") + ErrRetrieveCachedProgressFail = WrapStagedSyncError("retrieving cache progress for block hashes stage failed") + ErrRetrieveCachedHashProgressFail = WrapStagedSyncError("retrieving cache progress for block hashes stage failed") + ErrSaveBlockHashesProgressFail = WrapStagedSyncError("saving progress for block hashes stage failed") + ErrSaveCachedBlockHashesProgressFail = WrapStagedSyncError("saving cache progress for block hashes stage failed") + ErrSavingCacheLastBlockHashFail = WrapStagedSyncError("saving cache last block hash for block hashes stage failed") + ErrCachingBlockHashFail = WrapStagedSyncError("caching downloaded block hashes failed") + ErrCommitTransactionFail = WrapStagedSyncError("failed to write db commit") + ErrUnexpectedNumberOfBlocks = WrapStagedSyncError("unexpected number of block delivered") + ErrSavingBodiesProgressFail = WrapStagedSyncError("saving progress for block bodies stage failed") + ErrAddTasksToQueueFail = WrapStagedSyncError("cannot add task to queue") + ErrSavingCachedBodiesProgressFail = WrapStagedSyncError("saving cache progress for blocks stage failed") + ErrRetrievingCachedBodiesProgressFail = WrapStagedSyncError("retrieving cache progress for blocks stage failed") + ErrNoConnectedPeers = WrapStagedSyncError("haven't connected to any peer yet!") + ErrNotEnoughConnectedPeers = WrapStagedSyncError("not enough connected peers") + ErrSaveStateProgressFail = WrapStagedSyncError("saving progress for block States stage failed") + ErrPruningCursorCreationFail = WrapStagedSyncError("failed to create cursor for pruning") + ErrInvalidBlockNumber = WrapStagedSyncError("invalid block number") + ErrInvalidBlockBytes = WrapStagedSyncError("invalid block bytes to insert into chain") + ErrAddTaskFailed = WrapStagedSyncError("cannot add task to queue") + ErrNodeNotEnoughBlockHashes = WrapStagedSyncError("some of the nodes didn't provide all block hashes") + ErrCachingBlocksFail = WrapStagedSyncError("caching downloaded block bodies failed") + ErrSaveBlocksFail = WrapStagedSyncError("save downloaded block bodies failed") + ErrStageNotFound = WrapStagedSyncError("stage not found") + ErrSomeNodesNotReady = WrapStagedSyncError("some nodes are not ready") + ErrSomeNodesBlockHashFail = WrapStagedSyncError("some nodes failed to download block hashes") + ErrMaxPeerHeightFail = WrapStagedSyncError("get max peer height failed") +) + +// WrapStagedSyncError wraps errors for staged sync and returns error object +func WrapStagedSyncError(context string) error { + return fmt.Errorf("[STAGED_STREAM_SYNC]: %s", context) +} + +// WrapStagedSyncMsg wraps message for staged sync and returns string +func WrapStagedSyncMsg(context string) string { + return fmt.Sprintf("[STAGED_STREAM_SYNC]: %s", context) +} diff --git a/api/service/stagedstreamsync/helpers.go b/api/service/stagedstreamsync/helpers.go new file mode 100644 index 0000000000..cd6fd8f6f0 --- /dev/null +++ b/api/service/stagedstreamsync/helpers.go @@ -0,0 +1,114 @@ +package stagedstreamsync + +import ( + "encoding/binary" + "fmt" + "math" + + "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/harmony/core/types" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/pkg/errors" +) + +func marshalData(blockNumber uint64) []byte { + return encodeBigEndian(blockNumber) +} + +func unmarshalData(data []byte) (uint64, error) { + if len(data) == 0 { + return 0, nil + } + if len(data) < 8 { + return 0, fmt.Errorf("value must be at least 8 bytes, got %d", len(data)) + } + return binary.BigEndian.Uint64(data[:8]), nil +} + +func encodeBigEndian(n uint64) []byte { + var v [8]byte + binary.BigEndian.PutUint64(v[:], n) + return v[:] +} + +func divideCeil(x, y int) int { + fVal := float64(x) / float64(y) + return int(math.Ceil(fVal)) +} + +// computeBlockNumberByMaxVote computes the target block number by max vote. +func computeBlockNumberByMaxVote(votes map[sttypes.StreamID]uint64) uint64 { + var ( + nm = make(map[uint64]int) + res uint64 + maxCnt int + ) + for _, bn := range votes { + _, ok := nm[bn] + if !ok { + nm[bn] = 0 + } + nm[bn]++ + cnt := nm[bn] + + if cnt > maxCnt || (cnt == maxCnt && bn > res) { + res = bn + maxCnt = cnt + } + } + return res +} + +func checkGetBlockByHashesResult(blocks []*types.Block, hashes []common.Hash) error { + if len(blocks) != len(hashes) { + return errors.New("unexpected number of getBlocksByHashes result") + } + for i, block := range blocks { + if block == nil { + return errors.New("nil block found") + } + if block.Hash() != hashes[i] { + return fmt.Errorf("unexpected block hash: %x / %x", block.Hash(), hashes[i]) + } + } + return nil +} + +func countHashMaxVote(m map[sttypes.StreamID]common.Hash, whitelist map[sttypes.StreamID]struct{}) (common.Hash, map[sttypes.StreamID]struct{}) { + var ( + voteM = make(map[common.Hash]int) + res common.Hash + maxCnt = 0 + ) + + for st, h := range m { + if len(whitelist) != 0 { + if _, ok := whitelist[st]; !ok { + continue + } + } + if _, ok := voteM[h]; !ok { + voteM[h] = 0 + } + voteM[h]++ + if voteM[h] > maxCnt { + maxCnt = voteM[h] + res = h + } + } + + nextWl := make(map[sttypes.StreamID]struct{}) + for st, h := range m { + if h != res { + continue + } + if len(whitelist) != 0 { + if _, ok := whitelist[st]; ok { + nextWl[st] = struct{}{} + } + } else { + nextWl[st] = struct{}{} + } + } + return res, nextWl +} diff --git a/api/service/stagedstreamsync/metric.go b/api/service/stagedstreamsync/metric.go new file mode 100644 index 0000000000..1bc11a227f --- /dev/null +++ b/api/service/stagedstreamsync/metric.go @@ -0,0 +1,98 @@ +package stagedstreamsync + +import ( + "fmt" + + prom "github.com/harmony-one/harmony/api/service/prometheus" + "github.com/prometheus/client_golang/prometheus" +) + +func init() { + prom.PromRegistry().MustRegister( + consensusTriggeredDownloadCounterVec, + longRangeSyncedBlockCounterVec, + longRangeFailInsertedBlockCounterVec, + numShortRangeCounterVec, + numFailedDownloadCounterVec, + numBlocksInsertedShortRangeHistogramVec, + numBlocksInsertedBeaconHelperCounter, + ) +} + +var ( + consensusTriggeredDownloadCounterVec = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "hmy", + Subsystem: "downloader", + Name: "consensus_trigger", + Help: "number of times consensus triggered download task", + }, + []string{"ShardID"}, + ) + + longRangeSyncedBlockCounterVec = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "hmy", + Subsystem: "downloader", + Name: "num_blocks_synced_long_range", + Help: "number of blocks synced in long range sync", + }, + []string{"ShardID"}, + ) + + longRangeFailInsertedBlockCounterVec = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "hmy", + Subsystem: "downloader", + Name: "num_blocks_failed_long_range", + Help: "number of blocks failed to insert into change in long range sync", + }, + []string{"ShardID", "error"}, + ) + + numShortRangeCounterVec = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "hmy", + Subsystem: "downloader", + Name: "num_short_range", + Help: "number of short range sync is triggered", + }, + []string{"ShardID"}, + ) + + numFailedDownloadCounterVec = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "hmy", + Subsystem: "downloader", + Name: "failed_download", + Help: "number of downloading is failed", + }, + []string{"ShardID", "error"}, + ) + + numBlocksInsertedShortRangeHistogramVec = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "hmy", + Subsystem: "downloader", + Name: "num_blocks_inserted_short_range", + Help: "number of blocks inserted for each short range sync", + // Buckets: 0, 1, 2, 4, +INF (capped at 10) + Buckets: prometheus.ExponentialBuckets(0.5, 2, 5), + }, + []string{"ShardID"}, + ) + + numBlocksInsertedBeaconHelperCounter = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "hmy", + Subsystem: "downloader", + Name: "num_blocks_inserted_beacon_helper", + Help: "number of blocks inserted from beacon helper", + }, + ) +) + +func (d *Downloader) promLabels() prometheus.Labels { + sid := d.bc.ShardID() + return prometheus.Labels{"ShardID": fmt.Sprintf("%d", sid)} +} diff --git a/api/service/stagedstreamsync/service.go b/api/service/stagedstreamsync/service.go new file mode 100644 index 0000000000..46b182fb53 --- /dev/null +++ b/api/service/stagedstreamsync/service.go @@ -0,0 +1,30 @@ +package stagedstreamsync + +import ( + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/p2p" +) + +// StagedStreamSyncService is simply a adapter of downloaders, which support block synchronization +type StagedStreamSyncService struct { + Downloaders *Downloaders +} + +// NewService creates a new downloader service +func NewService(host p2p.Host, bcs []core.BlockChain, config Config) *StagedStreamSyncService { + return &StagedStreamSyncService{ + Downloaders: NewDownloaders(host, bcs, config), + } +} + +// Start starts the service +func (s *StagedStreamSyncService) Start() error { + s.Downloaders.Start() + return nil +} + +// Stop stops the service +func (s *StagedStreamSyncService) Stop() error { + s.Downloaders.Close() + return nil +} diff --git a/api/service/stagedstreamsync/short_range_helper.go b/api/service/stagedstreamsync/short_range_helper.go new file mode 100644 index 0000000000..f8dd2d28ea --- /dev/null +++ b/api/service/stagedstreamsync/short_range_helper.go @@ -0,0 +1,221 @@ +package stagedstreamsync + +import ( + "context" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/harmony/core/types" + syncProto "github.com/harmony-one/harmony/p2p/stream/protocols/sync" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/pkg/errors" + "github.com/rs/zerolog" +) + +type srHelper struct { + syncProtocol syncProtocol + + ctx context.Context + config Config + logger zerolog.Logger +} + +func (sh *srHelper) getHashChain(bns []uint64) ([]common.Hash, []sttypes.StreamID, error) { + results := newBlockHashResults(bns) + + var wg sync.WaitGroup + wg.Add(sh.config.Concurrency) + + for i := 0; i != sh.config.Concurrency; i++ { + go func(index int) { + defer wg.Done() + + hashes, stid, err := sh.doGetBlockHashesRequest(bns) + if err != nil { + sh.logger.Warn().Err(err).Str("StreamID", string(stid)). + Msg(WrapStagedSyncMsg("doGetBlockHashes return error")) + return + } + results.addResult(hashes, stid) + }(i) + } + wg.Wait() + + select { + case <-sh.ctx.Done(): + sh.logger.Info().Err(sh.ctx.Err()).Int("num blocks", results.numBlocksWithResults()). + Msg(WrapStagedSyncMsg("short range sync get hashes timed out")) + return nil, nil, sh.ctx.Err() + default: + } + + hashChain, wl := results.computeLongestHashChain() + sh.logger.Info().Int("hashChain size", len(hashChain)).Int("whitelist", len(wl)). + Msg(WrapStagedSyncMsg("computeLongestHashChain result")) + return hashChain, wl, nil +} + +func (sh *srHelper) getBlocksChain(bns []uint64) ([]*types.Block, sttypes.StreamID, error) { + return sh.doGetBlocksByNumbersRequest(bns) +} + +func (sh *srHelper) getBlocksByHashes(hashes []common.Hash, whitelist []sttypes.StreamID) ([]*types.Block, []sttypes.StreamID, error) { + ctx, cancel := context.WithCancel(sh.ctx) + defer cancel() + m := newGetBlocksByHashManager(hashes, whitelist) + + var ( + wg sync.WaitGroup + gErr error + errLock sync.Mutex + ) + + concurrency := sh.config.Concurrency + if concurrency > m.numRequests() { + concurrency = m.numRequests() + } + + wg.Add(concurrency) + for i := 0; i != concurrency; i++ { + go func(index int) { + defer wg.Done() + defer cancel() // it's ok to cancel context more than once + + for { + if m.isDone() { + return + } + hashes, wl, err := m.getNextHashes() + if err != nil { + errLock.Lock() + gErr = err + errLock.Unlock() + return + } + if len(hashes) == 0 { + select { + case <-time.After(200 * time.Millisecond): + continue + case <-ctx.Done(): + return + } + } + blocks, stid, err := sh.doGetBlocksByHashesRequest(ctx, hashes, wl) + if err != nil { + sh.logger.Warn().Err(err). + Str("StreamID", string(stid)). + Int("hashes", len(hashes)). + Int("index", index). + Msg(WrapStagedSyncMsg("getBlocksByHashes worker failed")) + m.handleResultError(hashes, stid) + } else { + m.addResult(hashes, blocks, stid) + } + } + }(i) + } + wg.Wait() + + if gErr != nil { + return nil, nil, gErr + } + select { + case <-sh.ctx.Done(): + res, _, _ := m.getResults() + sh.logger.Info().Err(sh.ctx.Err()).Int("num blocks", len(res)). + Msg(WrapStagedSyncMsg("short range sync get blocks timed out")) + return nil, nil, sh.ctx.Err() + default: + } + + return m.getResults() +} + +func (sh *srHelper) checkPrerequisites() error { + if sh.syncProtocol.NumStreams() < sh.config.Concurrency { + return errors.New("not enough streams") + } + return nil +} + +func (sh *srHelper) prepareBlockHashNumbers(curNumber uint64, count int) []uint64 { + + n := count + if count > BlockHashesPerRequest { + n = BlockHashesPerRequest + } + res := make([]uint64, 0, n) + + for bn := curNumber + 1; bn <= curNumber+uint64(n); bn++ { + res = append(res, bn) + } + return res +} + +func (sh *srHelper) doGetBlockHashesRequest(bns []uint64) ([]common.Hash, sttypes.StreamID, error) { + ctx, cancel := context.WithTimeout(sh.ctx, 1*time.Second) + defer cancel() + + hashes, stid, err := sh.syncProtocol.GetBlockHashes(ctx, bns) + if err != nil { + sh.logger.Warn().Err(err). + Interface("block numbers", bns). + Str("stream", string(stid)). + Msg(WrapStagedSyncMsg("failed to doGetBlockHashesRequest")) + return nil, stid, err + } + if len(hashes) != len(bns) { + err := errors.New("unexpected get block hashes result delivered") + sh.logger.Warn().Err(err). + Str("stream", string(stid)). + Msg(WrapStagedSyncMsg("failed to doGetBlockHashesRequest")) + sh.syncProtocol.StreamFailed(stid, "unexpected get block hashes result delivered") + return nil, stid, err + } + return hashes, stid, nil +} + +func (sh *srHelper) doGetBlocksByNumbersRequest(bns []uint64) ([]*types.Block, sttypes.StreamID, error) { + ctx, cancel := context.WithTimeout(sh.ctx, 10*time.Second) + defer cancel() + + blocks, stid, err := sh.syncProtocol.GetBlocksByNumber(ctx, bns) + if err != nil { + sh.logger.Warn().Err(err).Str("stream", string(stid)).Msg(WrapStagedSyncMsg("failed to doGetBlockHashesRequest")) + return nil, stid, err + } + return blocks, stid, nil +} + +func (sh *srHelper) doGetBlocksByHashesRequest(ctx context.Context, hashes []common.Hash, wl []sttypes.StreamID) ([]*types.Block, sttypes.StreamID, error) { + ctx, cancel := context.WithTimeout(sh.ctx, 10*time.Second) + defer cancel() + + blocks, stid, err := sh.syncProtocol.GetBlocksByHashes(ctx, hashes, + syncProto.WithWhitelist(wl)) + if err != nil { + sh.logger.Warn().Err(err).Str("stream", string(stid)).Msg("failed to getBlockByHashes") + return nil, stid, err + } + if err := checkGetBlockByHashesResult(blocks, hashes); err != nil { + sh.logger.Warn().Err(err).Str("stream", string(stid)).Msg(WrapStagedSyncMsg("failed to getBlockByHashes")) + sh.syncProtocol.StreamFailed(stid, "failed to getBlockByHashes") + return nil, stid, err + } + return blocks, stid, nil +} + +func (sh *srHelper) removeStreams(sts []sttypes.StreamID) { + for _, st := range sts { + sh.syncProtocol.RemoveStream(st) + } +} + +// blameAllStreams only not to blame all whitelisted streams when the it's not the last block signature verification failed. +func (sh *srHelper) blameAllStreams(blocks types.Blocks, errIndex int, err error) bool { + if errors.As(err, &emptySigVerifyErr) && errIndex == len(blocks)-1 { + return false + } + return true +} diff --git a/api/service/stagedstreamsync/stage.go b/api/service/stagedstreamsync/stage.go new file mode 100644 index 0000000000..8f2040a789 --- /dev/null +++ b/api/service/stagedstreamsync/stage.go @@ -0,0 +1,112 @@ +package stagedstreamsync + +import ( + "context" + "errors" + + "github.com/ethereum/go-ethereum/common" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/ledgerwatch/erigon-lib/kv" +) + +type ExecFunc func(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error + +type StageHandler interface { + // Exec is the execution function for the stage to move forward. + // * firstCycle - is it the first cycle of syncing. + // * invalidBlockRevert - whether the execution is to solve the invalid block + // * s - is the current state of the stage and contains stage data. + // * reverter - if the stage needs to cause reverting, `reverter` methods can be used. + Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error + + // Revert is the reverting logic of the stage. + // * firstCycle - is it the first cycle of syncing. + // * u - contains information about the revert itself. + // * s - represents the state of this stage at the beginning of revert. + Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) error + + // CleanUp is the execution function for the stage to prune old data. + // * firstCycle - is it the first cycle of syncing. + // * p - is the current state of the stage and contains stage data. + CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) error + + // SetStageContext updates the context for stage + SetStageContext(ctx context.Context) +} + +// Stage is a single sync stage in staged sync. +type Stage struct { + // ID of the sync stage. Should not be empty and should be unique. It is recommended to prefix it with reverse domain to avoid clashes (`com.example.my-stage`). + ID SyncStageID + // Handler handles the logic for the stage + Handler StageHandler + // Description is a string that is shown in the logs. + Description string + // DisabledDescription shows in the log with a message if the stage is disabled. Here, you can show which command line flags should be provided to enable the page. + DisabledDescription string + // Disabled defines if the stage is disabled. It sets up when the stage is build by its `StageBuilder`. + Disabled bool +} + +var ErrStopped = errors.New("stopped") +var ErrRevert = errors.New("unwound") + +// StageState is the state of the stage. +type StageState struct { + state *StagedStreamSync + ID SyncStageID + BlockNumber uint64 // BlockNumber is the current block number of the stage at the beginning of the state execution. +} + +func (s *StageState) LogPrefix() string { return s.state.LogPrefix() } + +func (s *StageState) CurrentStageProgress(db kv.Getter) (uint64, error) { + return GetStageProgress(db, s.ID, s.state.isBeacon) +} + +func (s *StageState) StageProgress(db kv.Getter, id SyncStageID) (uint64, error) { + return GetStageProgress(db, id, s.state.isBeacon) +} + +// Update updates the stage state (current block number) in the database. Can be called multiple times during stage execution. +func (s *StageState) Update(db kv.Putter, newBlockNum uint64) error { + return SaveStageProgress(db, s.ID, s.state.isBeacon, newBlockNum) +} +func (s *StageState) UpdateCleanUp(db kv.Putter, blockNum uint64) error { + return SaveStageCleanUpProgress(db, s.ID, s.state.isBeacon, blockNum) +} + +// Reverter allows the stage to cause an revert. +type Reverter interface { + // RevertTo begins staged sync revert to the specified block. + RevertTo(revertPoint uint64, invalidBlockNumber uint64, invalidBlockHash common.Hash, invalidBlockStreamID sttypes.StreamID) +} + +// RevertState contains the information about revert. +type RevertState struct { + ID SyncStageID + RevertPoint uint64 // RevertPoint is the block to revert to. + state *StagedStreamSync +} + +func (u *RevertState) LogPrefix() string { return u.state.LogPrefix() } + +// Done updates the DB state of the stage. +func (u *RevertState) Done(db kv.Putter) error { + return SaveStageProgress(db, u.ID, u.state.isBeacon, u.RevertPoint) +} + +type CleanUpState struct { + ID SyncStageID + ForwardProgress uint64 // progress of stage forward move + CleanUpProgress uint64 // progress of stage prune move. after sync cycle it become equal to ForwardProgress by Done() method + state *StagedStreamSync +} + +func (s *CleanUpState) LogPrefix() string { return s.state.LogPrefix() + " CleanUp" } +func (s *CleanUpState) Done(db kv.Putter) error { + return SaveStageCleanUpProgress(db, s.ID, s.state.isBeacon, s.ForwardProgress) +} +func (s *CleanUpState) DoneAt(db kv.Putter, blockNum uint64) error { + return SaveStageCleanUpProgress(db, s.ID, s.state.isBeacon, blockNum) +} diff --git a/api/service/stagedstreamsync/stage_bodies.go b/api/service/stagedstreamsync/stage_bodies.go new file mode 100644 index 0000000000..1f122b0d9c --- /dev/null +++ b/api/service/stagedstreamsync/stage_bodies.go @@ -0,0 +1,420 @@ +package stagedstreamsync + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/internal/utils" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/pkg/errors" +) + +type StageBodies struct { + configs StageBodiesCfg +} +type StageBodiesCfg struct { + ctx context.Context + bc core.BlockChain + db kv.RwDB + blockDBs []kv.RwDB + concurrency int + protocol syncProtocol + isBeacon bool + logProgress bool +} + +func NewStageBodies(cfg StageBodiesCfg) *StageBodies { + return &StageBodies{ + configs: cfg, + } +} + +func NewStageBodiesCfg(ctx context.Context, bc core.BlockChain, db kv.RwDB, blockDBs []kv.RwDB, concurrency int, protocol syncProtocol, isBeacon bool, logProgress bool) StageBodiesCfg { + return StageBodiesCfg{ + ctx: ctx, + bc: bc, + db: db, + blockDBs: blockDBs, + concurrency: concurrency, + protocol: protocol, + isBeacon: isBeacon, + logProgress: logProgress, + } +} + +func (b *StageBodies) SetStageContext(ctx context.Context) { + b.configs.ctx = ctx +} + +// Exec progresses Bodies stage in the forward direction +func (b *StageBodies) Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) (err error) { + + useInternalTx := tx == nil + + if invalidBlockRevert { + return b.redownloadBadBlock(s) + } + + // for short range sync, skip this stage + if !s.state.initSync { + return nil + } + + maxHeight := s.state.status.targetBN + currentHead := b.configs.bc.CurrentBlock().NumberU64() + if currentHead >= maxHeight { + return nil + } + currProgress := uint64(0) + targetHeight := s.state.currentCycle.TargetHeight + // isBeacon := s.state.isBeacon + // isLastCycle := targetHeight >= maxHeight + + if errV := CreateView(b.configs.ctx, b.configs.db, tx, func(etx kv.Tx) error { + if currProgress, err = s.CurrentStageProgress(etx); err != nil { + return err + } + return nil + }); errV != nil { + return errV + } + + if currProgress == 0 { + if err := b.cleanAllBlockDBs(); err != nil { + return err + } + currProgress = currentHead + } + + if currProgress >= targetHeight { + return nil + } + + // size := uint64(0) + startTime := time.Now() + // startBlock := currProgress + if b.configs.logProgress { + fmt.Print("\033[s") // save the cursor position + } + + if useInternalTx { + var err error + tx, err = b.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + // Fetch blocks from neighbors + s.state.gbm = newBlockDownloadManager(tx, b.configs.bc, targetHeight, s.state.logger) + + // Setup workers to fetch blocks from remote node + var wg sync.WaitGroup + + for i := 0; i != s.state.config.Concurrency; i++ { + wg.Add(1) + go b.runBlockWorkerLoop(s.state.gbm, &wg, i, startTime) + } + + wg.Wait() + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + + return nil +} + +// runBlockWorkerLoop creates a work loop for download blocks +func (b *StageBodies) runBlockWorkerLoop(gbm *blockDownloadManager, wg *sync.WaitGroup, loopID int, startTime time.Time) { + + currentBlock := int(b.configs.bc.CurrentBlock().NumberU64()) + + defer wg.Done() + + for { + select { + case <-b.configs.ctx.Done(): + return + default: + } + batch := gbm.GetNextBatch() + if len(batch) == 0 { + select { + case <-b.configs.ctx.Done(): + return + case <-time.After(100 * time.Millisecond): + return + } + } + + blockBytes, sigBytes, stid, err := b.downloadRawBlocks(batch) + if err != nil { + if !errors.Is(err, context.Canceled) { + b.configs.protocol.StreamFailed(stid, "downloadRawBlocks failed") + } + utils.Logger().Error(). + Err(err). + Str("stream", string(stid)). + Interface("block numbers", batch). + Msg(WrapStagedSyncMsg("downloadRawBlocks failed")) + err = errors.Wrap(err, "request error") + gbm.HandleRequestError(batch, err, stid) + } else { + if err = b.saveBlocks(gbm.tx, batch, blockBytes, sigBytes, loopID, stid); err != nil { + panic("[STAGED_STREAM_SYNC] saving downloaded blocks to db failed.") + } + gbm.HandleRequestResult(batch, blockBytes, sigBytes, loopID, stid) + if b.configs.logProgress { + //calculating block download speed + dt := time.Now().Sub(startTime).Seconds() + speed := float64(0) + if dt > 0 { + speed = float64(len(gbm.bdd)) / dt + } + blockSpeed := fmt.Sprintf("%.2f", speed) + + fmt.Print("\033[u\033[K") // restore the cursor position and clear the line + fmt.Println("downloaded blocks:", currentBlock+len(gbm.bdd), "/", int(gbm.targetBN), "(", blockSpeed, "blocks/s", ")") + } + } + } +} + +// redownloadBadBlock tries to redownload the bad block from other streams +func (b *StageBodies) redownloadBadBlock(s *StageState) error { + + batch := make([]uint64, 1) + batch = append(batch, s.state.invalidBlock.Number) + + for { + if b.configs.protocol.NumStreams() == 0 { + return errors.Errorf("re-download bad block from all streams failed") + } + blockBytes, sigBytes, stid, err := b.downloadRawBlocks(batch) + if err != nil { + if !errors.Is(err, context.Canceled) { + b.configs.protocol.StreamFailed(stid, "tried to re-download bad block from this stream, but downloadRawBlocks failed") + } + continue + } + isOneOfTheBadStreams := false + for _, id := range s.state.invalidBlock.StreamID { + if id == stid { + b.configs.protocol.RemoveStream(stid) + isOneOfTheBadStreams = true + break + } + } + if isOneOfTheBadStreams { + continue + } + s.state.gbm.SetDownloadDetails(batch, 0, stid) + if errU := b.configs.blockDBs[0].Update(context.Background(), func(tx kv.RwTx) error { + if err = b.saveBlocks(tx, batch, blockBytes, sigBytes, 0, stid); err != nil { + return errors.Errorf("[STAGED_STREAM_SYNC] saving re-downloaded bad block to db failed.") + } + return nil + }); errU != nil { + continue + } + break + } + return nil +} + +func (b *StageBodies) downloadBlocks(bns []uint64) ([]*types.Block, sttypes.StreamID, error) { + ctx, cancel := context.WithTimeout(b.configs.ctx, 10*time.Second) + defer cancel() + + blocks, stid, err := b.configs.protocol.GetBlocksByNumber(ctx, bns) + if err != nil { + return nil, stid, err + } + if err := validateGetBlocksResult(bns, blocks); err != nil { + return nil, stid, err + } + return blocks, stid, nil +} + +func (b *StageBodies) downloadRawBlocks(bns []uint64) ([][]byte, [][]byte, sttypes.StreamID, error) { + ctx, cancel := context.WithTimeout(b.configs.ctx, 10*time.Second) + defer cancel() + + return b.configs.protocol.GetRawBlocksByNumber(ctx, bns) +} + +func validateGetBlocksResult(requested []uint64, result []*types.Block) error { + if len(result) != len(requested) { + return fmt.Errorf("unexpected number of blocks delivered: %v / %v", len(result), len(requested)) + } + for i, block := range result { + if block != nil && block.NumberU64() != requested[i] { + return fmt.Errorf("block with unexpected number delivered: %v / %v", block.NumberU64(), requested[i]) + } + } + return nil +} + +// saveBlocks saves the blocks into db +func (b *StageBodies) saveBlocks(tx kv.RwTx, bns []uint64, blockBytes [][]byte, sigBytes [][]byte, loopID int, stid sttypes.StreamID) error { + + tx, err := b.configs.blockDBs[loopID].BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + + for i := uint64(0); i < uint64(len(blockBytes)); i++ { + block := blockBytes[i] + sig := sigBytes[i] + if block == nil { + continue + } + + blkKey := marshalData(bns[i]) + + if err := tx.Put(BlocksBucket, blkKey, block); err != nil { + utils.Logger().Error(). + Err(err). + Uint64("block height", bns[i]). + Msg("[STAGED_STREAM_SYNC] adding block to db failed") + return err + } + // sigKey := []byte("s" + string(bns[i])) + if err := tx.Put(BlockSignaturesBucket, blkKey, sig); err != nil { + utils.Logger().Error(). + Err(err). + Uint64("block height", bns[i]). + Msg("[STAGED_STREAM_SYNC] adding block sig to db failed") + return err + } + } + + if err := tx.Commit(); err != nil { + return err + } + + return nil +} + +func (b *StageBodies) saveProgress(s *StageState, progress uint64, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + var err error + tx, err = b.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + // save progress + if err = s.Update(tx, progress); err != nil { + utils.Logger().Error(). + Err(err). + Msgf("[STAGED_SYNC] saving progress for block bodies stage failed") + return ErrSavingBodiesProgressFail + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + return nil +} + +func (b *StageBodies) cleanBlocksDB(loopID int) (err error) { + + tx, errb := b.configs.blockDBs[loopID].BeginRw(b.configs.ctx) + if errb != nil { + return errb + } + defer tx.Rollback() + + // clean block bodies db + if err = tx.ClearBucket(BlocksBucket); err != nil { + utils.Logger().Error(). + Err(err). + Msgf("[STAGED_STREAM_SYNC] clear blocks bucket after revert failed") + return err + } + // clean block signatures db + if err = tx.ClearBucket(BlockSignaturesBucket); err != nil { + utils.Logger().Error(). + Err(err). + Msgf("[STAGED_STREAM_SYNC] clear block signatures bucket after revert failed") + return err + } + + if err = tx.Commit(); err != nil { + return err + } + + return nil +} + +func (b *StageBodies) cleanAllBlockDBs() (err error) { + //clean all blocks DBs + for i := 0; i < b.configs.concurrency; i++ { + if err := b.cleanBlocksDB(i); err != nil { + return err + } + } + return nil +} + +func (b *StageBodies) Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { + + //clean all blocks DBs + if err := b.cleanAllBlockDBs(); err != nil { + return err + } + + useInternalTx := tx == nil + if useInternalTx { + tx, err = b.configs.db.BeginRw(b.configs.ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + // save progress + currentHead := b.configs.bc.CurrentBlock().NumberU64() + if err = s.Update(tx, currentHead); err != nil { + utils.Logger().Error(). + Err(err). + Msgf("[STAGED_SYNC] saving progress for block bodies stage after revert failed") + return err + } + + if err = u.Done(tx); err != nil { + return err + } + + if useInternalTx { + if err = tx.Commit(); err != nil { + return err + } + } + return nil +} + +func (b *StageBodies) CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { + + //clean all blocks DBs + if err := b.cleanAllBlockDBs(); err != nil { + return err + } + + return nil +} diff --git a/api/service/stagedstreamsync/stage_epoch.go b/api/service/stagedstreamsync/stage_epoch.go new file mode 100644 index 0000000000..6320d82175 --- /dev/null +++ b/api/service/stagedstreamsync/stage_epoch.go @@ -0,0 +1,198 @@ +package stagedstreamsync + +import ( + "context" + + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/shard" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/pkg/errors" +) + +type StageEpoch struct { + configs StageEpochCfg +} + +type StageEpochCfg struct { + ctx context.Context + bc core.BlockChain + db kv.RwDB +} + +func NewStageEpoch(cfg StageEpochCfg) *StageEpoch { + return &StageEpoch{ + configs: cfg, + } +} + +func NewStageEpochCfg(ctx context.Context, bc core.BlockChain, db kv.RwDB) StageEpochCfg { + return StageEpochCfg{ + ctx: ctx, + bc: bc, + db: db, + } +} + +func (sr *StageEpoch) SetStageContext(ctx context.Context) { + sr.configs.ctx = ctx +} + +func (sr *StageEpoch) Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error { + + // no need to update epoch chain if we are redoing the stages because of bad block + if invalidBlockRevert { + return nil + } + // for long range sync, skip this stage + if s.state.initSync { + return nil + } + + if _, ok := sr.configs.bc.(*core.EpochChain); !ok { + return nil + } + + // doShortRangeSyncForEpochSync + n, err := sr.doShortRangeSyncForEpochSync(s) + s.state.inserted = n + if err != nil { + return err + } + + useInternalTx := tx == nil + if useInternalTx { + var err error + tx, err = sr.configs.db.BeginRw(sr.configs.ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + + return nil +} + +func (sr *StageEpoch) doShortRangeSyncForEpochSync(s *StageState) (int, error) { + + numShortRangeCounterVec.With(s.state.promLabels()).Inc() + + srCtx, cancel := context.WithTimeout(s.state.ctx, ShortRangeTimeout) + defer cancel() + + //TODO: merge srHelper with StageEpochConfig + sh := &srHelper{ + syncProtocol: s.state.protocol, + ctx: srCtx, + config: s.state.config, + logger: utils.Logger().With().Str("mode", "epoch chain short range").Logger(), + } + + if err := sh.checkPrerequisites(); err != nil { + return 0, errors.Wrap(err, "prerequisite") + } + curBN := s.state.bc.CurrentBlock().NumberU64() + bns := make([]uint64, 0, BlocksPerRequest) + // in epoch chain, we have only the last block of each epoch, so, the current + // block's epoch number shows the last epoch we have. We should start + // from next epoch then + loopEpoch := s.state.bc.CurrentHeader().Epoch().Uint64() + 1 + for len(bns) < BlocksPerRequest { + blockNum := shard.Schedule.EpochLastBlock(loopEpoch) + if blockNum > curBN { + bns = append(bns, blockNum) + } + loopEpoch = loopEpoch + 1 + } + + if len(bns) == 0 { + return 0, nil + } + + //////////////////////////////////////////////////////// + hashChain, whitelist, err := sh.getHashChain(bns) + if err != nil { + return 0, errors.Wrap(err, "getHashChain") + } + if len(hashChain) == 0 { + // short circuit for no sync is needed + return 0, nil + } + blocks, streamID, err := sh.getBlocksByHashes(hashChain, whitelist) + if err != nil { + utils.Logger().Warn().Err(err).Msg("epoch sync getBlocksByHashes failed") + if !errors.Is(err, context.Canceled) { + sh.removeStreams(whitelist) // Remote nodes cannot provide blocks with target hashes + } + return 0, errors.Wrap(err, "epoch sync getBlocksByHashes") + } + /////////////////////////////////////////////////////// + // TODO: check this + // blocks, streamID, err := sh.getBlocksChain(bns) + // if err != nil { + // return 0, errors.Wrap(err, "getHashChain") + // } + /////////////////////////////////////////////////////// + if len(blocks) == 0 { + // short circuit for no sync is needed + return 0, nil + } + + n, err := s.state.bc.InsertChain(blocks, true) + numBlocksInsertedShortRangeHistogramVec.With(s.state.promLabels()).Observe(float64(n)) + if err != nil { + utils.Logger().Info().Err(err).Int("blocks inserted", n).Msg("Insert block failed") + sh.removeStreams(streamID) // Data provided by remote nodes is corrupted + return n, err + } + if n > 0 { + utils.Logger().Info().Int("blocks inserted", n).Msg("Insert block success") + } + return n, nil +} + +func (sr *StageEpoch) Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = sr.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + if err = u.Done(tx); err != nil { + return err + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + return nil +} + +func (sr *StageEpoch) CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = sr.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + if useInternalTx { + if err = tx.Commit(); err != nil { + return err + } + } + return nil +} diff --git a/api/service/stagedstreamsync/stage_finish.go b/api/service/stagedstreamsync/stage_finish.go new file mode 100644 index 0000000000..9039b5588d --- /dev/null +++ b/api/service/stagedstreamsync/stage_finish.go @@ -0,0 +1,114 @@ +package stagedstreamsync + +import ( + "context" + + "github.com/ledgerwatch/erigon-lib/kv" +) + +type StageFinish struct { + configs StageFinishCfg +} + +type StageFinishCfg struct { + ctx context.Context + db kv.RwDB +} + +func NewStageFinish(cfg StageFinishCfg) *StageFinish { + return &StageFinish{ + configs: cfg, + } +} + +func NewStageFinishCfg(ctx context.Context, db kv.RwDB) StageFinishCfg { + return StageFinishCfg{ + ctx: ctx, + db: db, + } +} + +func (finish *StageFinish) SetStageContext(ctx context.Context) { + finish.configs.ctx = ctx +} + +func (finish *StageFinish) Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error { + useInternalTx := tx == nil + if useInternalTx { + var err error + tx, err = finish.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + // TODO: prepare indices (useful for RPC) and finalize + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + + return nil +} + +func (bh *StageFinish) clearBucket(tx kv.RwTx, isBeacon bool) error { + useInternalTx := tx == nil + if useInternalTx { + var err error + tx, err = bh.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + return nil +} + +func (finish *StageFinish) Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = finish.configs.db.BeginRw(finish.configs.ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + if err = u.Done(tx); err != nil { + return err + } + + if useInternalTx { + if err = tx.Commit(); err != nil { + return err + } + } + return nil +} + +func (finish *StageFinish) CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = finish.configs.db.BeginRw(finish.configs.ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + if useInternalTx { + if err = tx.Commit(); err != nil { + return err + } + } + return nil +} diff --git a/api/service/stagedstreamsync/stage_heads.go b/api/service/stagedstreamsync/stage_heads.go new file mode 100644 index 0000000000..1ddd285aa0 --- /dev/null +++ b/api/service/stagedstreamsync/stage_heads.go @@ -0,0 +1,157 @@ +package stagedstreamsync + +import ( + "context" + + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/internal/utils" + "github.com/ledgerwatch/erigon-lib/kv" +) + +type StageHeads struct { + configs StageHeadsCfg +} + +type StageHeadsCfg struct { + ctx context.Context + bc core.BlockChain + db kv.RwDB +} + +func NewStageHeads(cfg StageHeadsCfg) *StageHeads { + return &StageHeads{ + configs: cfg, + } +} + +func NewStageHeadersCfg(ctx context.Context, bc core.BlockChain, db kv.RwDB) StageHeadsCfg { + return StageHeadsCfg{ + ctx: ctx, + bc: bc, + db: db, + } +} + +func (heads *StageHeads) SetStageContext(ctx context.Context) { + heads.configs.ctx = ctx +} + +func (heads *StageHeads) Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error { + + // no need to update target if we are redoing the stages because of bad block + if invalidBlockRevert { + return nil + } + + // no need for short range sync + if !s.state.initSync { + return nil + } + + useInternalTx := tx == nil + if useInternalTx { + var err error + tx, err = heads.configs.db.BeginRw(heads.configs.ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + maxHeight := s.state.status.targetBN + maxBlocksPerSyncCycle := uint64(1024) // TODO: should be in config -> s.state.MaxBlocksPerSyncCycle + currentHeight := heads.configs.bc.CurrentBlock().NumberU64() + s.state.currentCycle.TargetHeight = maxHeight + targetHeight := uint64(0) + if errV := CreateView(heads.configs.ctx, heads.configs.db, tx, func(etx kv.Tx) (err error) { + if targetHeight, err = s.CurrentStageProgress(etx); err != nil { + return err + } + return nil + }); errV != nil { + return errV + } + + if currentHeight >= maxHeight { + utils.Logger().Info().Uint64("current number", currentHeight).Uint64("target number", maxHeight). + Msg(WrapStagedSyncMsg("early return of long range sync")) + return nil + } + + // if current height is ahead of target height, we need recalculate target height + if currentHeight >= targetHeight { + if maxHeight <= currentHeight { + return nil + } + utils.Logger().Info(). + Uint64("max blocks per sync cycle", maxBlocksPerSyncCycle). + Uint64("maxPeersHeight", maxHeight). + Msgf(WrapStagedSyncMsg("current height is ahead of target height, target height is readjusted to max peers height")) + targetHeight = maxHeight + } + + if targetHeight > maxHeight { + targetHeight = maxHeight + } + + if maxBlocksPerSyncCycle > 0 && targetHeight-currentHeight > maxBlocksPerSyncCycle { + targetHeight = currentHeight + maxBlocksPerSyncCycle + } + + s.state.currentCycle.TargetHeight = targetHeight + + if err := s.Update(tx, targetHeight); err != nil { + utils.Logger().Error(). + Err(err). + Msgf(WrapStagedSyncMsg("saving progress for headers stage failed")) + return err + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + + return nil +} + +func (heads *StageHeads) Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = heads.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + if err = u.Done(tx); err != nil { + return err + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + return nil +} + +func (heads *StageHeads) CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = heads.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + if useInternalTx { + if err = tx.Commit(); err != nil { + return err + } + } + return nil +} diff --git a/api/service/stagedstreamsync/stage_short_range.go b/api/service/stagedstreamsync/stage_short_range.go new file mode 100644 index 0000000000..de83beafa3 --- /dev/null +++ b/api/service/stagedstreamsync/stage_short_range.go @@ -0,0 +1,205 @@ +package stagedstreamsync + +import ( + "context" + + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/internal/utils" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/pkg/errors" +) + +type StageShortRange struct { + configs StageShortRangeCfg +} + +type StageShortRangeCfg struct { + ctx context.Context + bc core.BlockChain + db kv.RwDB +} + +func NewStageShortRange(cfg StageShortRangeCfg) *StageShortRange { + return &StageShortRange{ + configs: cfg, + } +} + +func NewStageShortRangeCfg(ctx context.Context, bc core.BlockChain, db kv.RwDB) StageShortRangeCfg { + return StageShortRangeCfg{ + ctx: ctx, + bc: bc, + db: db, + } +} + +func (sr *StageShortRange) SetStageContext(ctx context.Context) { + sr.configs.ctx = ctx +} + +func (sr *StageShortRange) Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error { + + // no need to do short range if we are redoing the stages because of bad block + if invalidBlockRevert { + return nil + } + + // for long range sync, skip this stage + if s.state.initSync { + return nil + } + + if _, ok := sr.configs.bc.(*core.EpochChain); ok { + return nil + } + + curBN := sr.configs.bc.CurrentBlock().NumberU64() + if curBN >= s.state.status.targetBN { + return nil + } + + // do short range sync + n, err := sr.doShortRangeSync(s) + s.state.inserted = n + if err != nil { + return err + } + + useInternalTx := tx == nil + if useInternalTx { + var err error + tx, err = sr.configs.db.BeginRw(sr.configs.ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + + return nil +} + +// doShortRangeSync does the short range sync. +// Compared with long range sync, short range sync is more focused on syncing to the latest block. +// It consist of 3 steps: +// 1. Obtain the block hashes and compute the longest hash chain.. +// 2. Get blocks by hashes from computed hash chain. +// 3. Insert the blocks to blockchain. +func (sr *StageShortRange) doShortRangeSync(s *StageState) (int, error) { + + numShortRangeCounterVec.With(s.state.promLabels()).Inc() + + srCtx, cancel := context.WithTimeout(s.state.ctx, ShortRangeTimeout) + defer cancel() + + sh := &srHelper{ + syncProtocol: s.state.protocol, + ctx: srCtx, + config: s.state.config, + logger: utils.Logger().With().Str("mode", "short range").Logger(), + } + + if err := sh.checkPrerequisites(); err != nil { + return 0, errors.Wrap(err, "prerequisite") + } + curBN := sr.configs.bc.CurrentBlock().NumberU64() + blkCount := int(s.state.status.targetBN) - int(curBN) + blkNums := sh.prepareBlockHashNumbers(curBN, blkCount) + hashChain, whitelist, err := sh.getHashChain(blkNums) + if err != nil { + return 0, errors.Wrap(err, "getHashChain") + } + + if len(hashChain) == 0 { + // short circuit for no sync is needed + return 0, nil + } + + expEndBN := curBN + uint64(len(hashChain)) + utils.Logger().Info().Uint64("current number", curBN). + Uint64("target number", expEndBN). + Interface("hashChain", hashChain). + Msg("short range start syncing") + + s.state.status.setTargetBN(expEndBN) + + s.state.status.startSyncing() + defer func() { + utils.Logger().Info().Msg("short range finished syncing") + s.state.status.finishSyncing() + }() + + blocks, stids, err := sh.getBlocksByHashes(hashChain, whitelist) + if err != nil { + utils.Logger().Warn().Err(err).Msg("getBlocksByHashes failed") + if !errors.Is(err, context.Canceled) { + sh.removeStreams(whitelist) // Remote nodes cannot provide blocks with target hashes + } + return 0, errors.Wrap(err, "getBlocksByHashes") + } + + utils.Logger().Info().Int("num blocks", len(blocks)).Msg("getBlockByHashes result") + + n, err := verifyAndInsertBlocks(sr.configs.bc, blocks) + numBlocksInsertedShortRangeHistogramVec.With(s.state.promLabels()).Observe(float64(n)) + if err != nil { + utils.Logger().Warn().Err(err).Int("blocks inserted", n).Msg("Insert block failed") + if sh.blameAllStreams(blocks, n, err) { + sh.removeStreams(whitelist) // Data provided by remote nodes is corrupted + } else { + // It is the last block gives a wrong commit sig. Blame the provider of the last block. + st2Blame := stids[len(stids)-1] + sh.removeStreams([]sttypes.StreamID{st2Blame}) + } + return n, err + } + utils.Logger().Info().Err(err).Int("blocks inserted", n).Msg("Insert block success") + + return n, nil +} + +func (sr *StageShortRange) Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = sr.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + if err = u.Done(tx); err != nil { + return err + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + return nil +} + +func (sr *StageShortRange) CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = sr.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + if useInternalTx { + if err = tx.Commit(); err != nil { + return err + } + } + return nil +} diff --git a/api/service/stagedstreamsync/stage_state.go b/api/service/stagedstreamsync/stage_state.go new file mode 100644 index 0000000000..9eda042477 --- /dev/null +++ b/api/service/stagedstreamsync/stage_state.go @@ -0,0 +1,295 @@ +package stagedstreamsync + +import ( + "context" + "fmt" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/internal/utils" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/prometheus/client_golang/prometheus" + "github.com/rs/zerolog" +) + +type StageStates struct { + configs StageStatesCfg +} +type StageStatesCfg struct { + ctx context.Context + bc core.BlockChain + db kv.RwDB + blockDBs []kv.RwDB + concurrency int + logger zerolog.Logger + logProgress bool +} + +func NewStageStates(cfg StageStatesCfg) *StageStates { + return &StageStates{ + configs: cfg, + } +} + +func NewStageStatesCfg(ctx context.Context, + bc core.BlockChain, + db kv.RwDB, + blockDBs []kv.RwDB, + concurrency int, + logger zerolog.Logger, + logProgress bool) StageStatesCfg { + + return StageStatesCfg{ + ctx: ctx, + bc: bc, + db: db, + blockDBs: blockDBs, + concurrency: concurrency, + logger: logger, + logProgress: logProgress, + } +} + +func (stg *StageStates) SetStageContext(ctx context.Context) { + stg.configs.ctx = ctx +} + +// Exec progresses States stage in the forward direction +func (stg *StageStates) Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) (err error) { + + // for short range sync, skip this step + if !s.state.initSync { + return nil + } + + maxHeight := s.state.status.targetBN + currentHead := stg.configs.bc.CurrentBlock().NumberU64() + if currentHead >= maxHeight { + return nil + } + currProgress := stg.configs.bc.CurrentBlock().NumberU64() + targetHeight := s.state.currentCycle.TargetHeight + if currProgress >= targetHeight { + return nil + } + useInternalTx := tx == nil + if useInternalTx { + var err error + tx, err = stg.configs.db.BeginRw(stg.configs.ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + // isLastCycle := targetHeight >= maxHeight + startTime := time.Now() + startBlock := currProgress + pl := s.state.promLabels() + gbm := s.state.gbm + + // prepare db transactions + txs := make([]kv.RwTx, stg.configs.concurrency) + for i := 0; i < stg.configs.concurrency; i++ { + txs[i], err = stg.configs.blockDBs[i].BeginRw(context.Background()) + if err != nil { + return err + } + } + + defer func() { + for i := 0; i < stg.configs.concurrency; i++ { + txs[i].Rollback() + } + }() + + if stg.configs.logProgress { + fmt.Print("\033[s") // save the cursor position + } + + for i := currProgress + 1; i <= targetHeight; i++ { + blkKey := marshalData(i) + loopID, streamID := gbm.GetDownloadDetails(i) + + blockBytes, err := txs[loopID].GetOne(BlocksBucket, blkKey) + if err != nil { + return err + } + sigBytes, err := txs[loopID].GetOne(BlockSignaturesBucket, blkKey) + if err != nil { + return err + } + + // if block size is invalid, we have to break the updating state loop + // we don't need to do rollback, because the latest batch haven't added to chain yet + sz := len(blockBytes) + if sz <= 1 { + utils.Logger().Error(). + Uint64("block number", i). + Msg("block size invalid") + invalidBlockHash := common.Hash{} + s.state.protocol.StreamFailed(streamID, "zero bytes block is received from stream") + reverter.RevertTo(stg.configs.bc.CurrentBlock().NumberU64(), i, invalidBlockHash, streamID) + return ErrInvalidBlockBytes + } + + var block *types.Block + if err := rlp.DecodeBytes(blockBytes, &block); err != nil { + utils.Logger().Error(). + Uint64("block number", i). + Msg("block size invalid") + s.state.protocol.StreamFailed(streamID, "invalid block is received from stream") + invalidBlockHash := common.Hash{} + reverter.RevertTo(stg.configs.bc.CurrentBlock().NumberU64(), i, invalidBlockHash, streamID) + return ErrInvalidBlockBytes + } + if sigBytes != nil { + block.SetCurrentCommitSig(sigBytes) + } + + if block.NumberU64() != i { + s.state.protocol.StreamFailed(streamID, "invalid block with unmatched number is received from stream") + invalidBlockHash := block.Hash() + reverter.RevertTo(stg.configs.bc.CurrentBlock().NumberU64(), i, invalidBlockHash, streamID) + return ErrInvalidBlockNumber + } + + if err := verifyAndInsertBlock(stg.configs.bc, block); err != nil { + stg.configs.logger.Warn().Err(err).Uint64("cycle target block", targetHeight). + Uint64("block number", block.NumberU64()). + Msg(WrapStagedSyncMsg("insert blocks failed in long range")) + s.state.protocol.StreamFailed(streamID, "unverifiable invalid block is received from stream") + invalidBlockHash := block.Hash() + reverter.RevertTo(stg.configs.bc.CurrentBlock().NumberU64(), block.NumberU64(), invalidBlockHash, streamID) + pl["error"] = err.Error() + longRangeFailInsertedBlockCounterVec.With(pl).Inc() + return err + } + + if invalidBlockRevert { + if s.state.invalidBlock.Number == i { + s.state.invalidBlock.resolve() + } + } + + s.state.inserted++ + longRangeSyncedBlockCounterVec.With(pl).Inc() + + utils.Logger().Info(). + Uint64("blockHeight", block.NumberU64()). + Uint64("blockEpoch", block.Epoch().Uint64()). + Str("blockHex", block.Hash().Hex()). + Uint32("ShardID", block.ShardID()). + Msg("[STAGED_STREAM_SYNC] New Block Added to Blockchain") + + // update cur progress + currProgress = stg.configs.bc.CurrentBlock().NumberU64() + + for i, tx := range block.StakingTransactions() { + utils.Logger().Info(). + Msgf( + "StakingTxn %d: %s, %v", i, tx.StakingType().String(), tx.StakingMessage(), + ) + } + + // log the stage progress in console + if stg.configs.logProgress { + //calculating block speed + dt := time.Now().Sub(startTime).Seconds() + speed := float64(0) + if dt > 0 { + speed = float64(currProgress-startBlock) / dt + } + blockSpeed := fmt.Sprintf("%.2f", speed) + fmt.Print("\033[u\033[K") // restore the cursor position and clear the line + fmt.Println("insert blocks progress:", currProgress, "/", targetHeight, "(", blockSpeed, "blocks/s", ")") + } + + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + + return nil +} + +func (stg *StageStates) insertChain(gbm *blockDownloadManager, + protocol syncProtocol, + lbls prometheus.Labels, + targetBN uint64) { + +} + +func (stg *StageStates) saveProgress(s *StageState, tx kv.RwTx) (err error) { + + useInternalTx := tx == nil + if useInternalTx { + var err error + tx, err = stg.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + // save progress + if err = s.Update(tx, stg.configs.bc.CurrentBlock().NumberU64()); err != nil { + utils.Logger().Error(). + Err(err). + Msgf("[STAGED_SYNC] saving progress for block States stage failed") + return ErrSaveStateProgressFail + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + return nil +} + +func (stg *StageStates) Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = stg.configs.db.BeginRw(stg.configs.ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + if err = u.Done(tx); err != nil { + return err + } + + if useInternalTx { + if err = tx.Commit(); err != nil { + return err + } + } + return nil +} + +func (stg *StageStates) CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = stg.configs.db.BeginRw(stg.configs.ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + if useInternalTx { + if err = tx.Commit(); err != nil { + return err + } + } + return nil +} diff --git a/api/service/stagedstreamsync/staged_stream_sync.go b/api/service/stagedstreamsync/staged_stream_sync.go new file mode 100644 index 0000000000..1abd728498 --- /dev/null +++ b/api/service/stagedstreamsync/staged_stream_sync.go @@ -0,0 +1,597 @@ +package stagedstreamsync + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/event" + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/internal/utils" + syncproto "github.com/harmony-one/harmony/p2p/stream/protocols/sync" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/prometheus/client_golang/prometheus" + "github.com/rs/zerolog" +) + +type InvalidBlock struct { + Active bool + Number uint64 + Hash common.Hash + IsLogged bool + StreamID []sttypes.StreamID +} + +func (ib *InvalidBlock) set(num uint64, hash common.Hash, resetBadStreams bool) { + ib.Active = true + ib.IsLogged = false + ib.Number = num + ib.Hash = hash + if resetBadStreams { + ib.StreamID = make([]sttypes.StreamID, 0) + } +} + +func (ib *InvalidBlock) resolve() { + ib.Active = false + ib.IsLogged = false + ib.Number = 0 + ib.Hash = common.Hash{} + ib.StreamID = ib.StreamID[:0] +} + +func (ib *InvalidBlock) addBadStream(bsID sttypes.StreamID) { + // only add uniques IDs + for _, stID := range ib.StreamID { + if stID == bsID { + return + } + } + ib.StreamID = append(ib.StreamID, bsID) +} + +type StagedStreamSync struct { + ctx context.Context + bc core.BlockChain + isBeacon bool + isExplorer bool + db kv.RwDB + protocol syncProtocol + gbm *blockDownloadManager // initialized when finished get block number + inserted int + config Config + logger zerolog.Logger + status status //TODO: merge this with currentSyncCycle + initSync bool // if sets to true, node start long range syncing + UseMemDB bool + + revertPoint *uint64 // used to run stages + prevRevertPoint *uint64 // used to get value from outside of staged sync after cycle (for example to notify RPCDaemon) + invalidBlock InvalidBlock + currentStage uint + LogProgress bool + currentCycle SyncCycle // current cycle + stages []*Stage + revertOrder []*Stage + pruningOrder []*Stage + timings []Timing + logPrefixes []string + + evtDownloadFinished event.Feed // channel for each download task finished + evtDownloadFinishedSubscribed bool + evtDownloadStarted event.Feed // channel for each download has started + evtDownloadStartedSubscribed bool +} + +// BlockWithSig the serialization structure for request DownloaderRequest_BLOCKWITHSIG +// The block is encoded as block + commit signature +type BlockWithSig struct { + Block *types.Block + CommitSigAndBitmap []byte +} + +type Timing struct { + isRevert bool + isCleanUp bool + stage SyncStageID + took time.Duration +} + +type SyncCycle struct { + Number uint64 + TargetHeight uint64 + lock sync.RWMutex +} + +func (s *StagedStreamSync) Len() int { return len(s.stages) } +func (s *StagedStreamSync) Context() context.Context { return s.ctx } +func (s *StagedStreamSync) Blockchain() core.BlockChain { return s.bc } +func (s *StagedStreamSync) DB() kv.RwDB { return s.db } +func (s *StagedStreamSync) IsBeacon() bool { return s.isBeacon } +func (s *StagedStreamSync) IsExplorer() bool { return s.isExplorer } +func (s *StagedStreamSync) LogPrefix() string { + if s == nil { + return "" + } + return s.logPrefixes[s.currentStage] +} +func (s *StagedStreamSync) PrevRevertPoint() *uint64 { return s.prevRevertPoint } + +func (s *StagedStreamSync) NewRevertState(id SyncStageID, revertPoint uint64) *RevertState { + return &RevertState{id, revertPoint, s} +} + +func (s *StagedStreamSync) CleanUpStageState(id SyncStageID, forwardProgress uint64, tx kv.Tx, db kv.RwDB) (*CleanUpState, error) { + var pruneProgress uint64 + var err error + + if errV := CreateView(context.Background(), db, tx, func(tx kv.Tx) error { + pruneProgress, err = GetStageCleanUpProgress(tx, id, s.isBeacon) + if err != nil { + return err + } + return nil + }); errV != nil { + return nil, errV + } + + return &CleanUpState{id, forwardProgress, pruneProgress, s}, nil +} + +func (s *StagedStreamSync) NextStage() { + if s == nil { + return + } + s.currentStage++ +} + +// IsBefore returns true if stage1 goes before stage2 in staged sync +func (s *StagedStreamSync) IsBefore(stage1, stage2 SyncStageID) bool { + idx1 := -1 + idx2 := -1 + for i, stage := range s.stages { + if stage.ID == stage1 { + idx1 = i + } + + if stage.ID == stage2 { + idx2 = i + } + } + + return idx1 < idx2 +} + +// IsAfter returns true if stage1 goes after stage2 in staged sync +func (s *StagedStreamSync) IsAfter(stage1, stage2 SyncStageID) bool { + idx1 := -1 + idx2 := -1 + for i, stage := range s.stages { + if stage.ID == stage1 { + idx1 = i + } + + if stage.ID == stage2 { + idx2 = i + } + } + + return idx1 > idx2 +} + +func (s *StagedStreamSync) RevertTo(revertPoint uint64, invalidBlockNumber uint64, invalidBlockHash common.Hash, invalidBlockStreamID sttypes.StreamID) { + utils.Logger().Info(). + Interface("invalidBlockNumber", invalidBlockNumber). + Interface("invalidBlockHash", invalidBlockHash). + Interface("invalidBlockStreamID", invalidBlockStreamID). + Uint64("revertPoint", revertPoint). + Msgf(WrapStagedSyncMsg("Reverting blocks")) + s.revertPoint = &revertPoint + if invalidBlockNumber > 0 || invalidBlockHash != (common.Hash{}) { + resetBadStreams := !s.invalidBlock.Active + s.invalidBlock.set(invalidBlockNumber, invalidBlockHash, resetBadStreams) + s.invalidBlock.addBadStream(invalidBlockStreamID) + } +} + +func (s *StagedStreamSync) Done() { + s.currentStage = uint(len(s.stages)) + s.revertPoint = nil +} + +func (s *StagedStreamSync) IsDone() bool { + return s.currentStage >= uint(len(s.stages)) && s.revertPoint == nil +} + +func (s *StagedStreamSync) SetCurrentStage(id SyncStageID) error { + for i, stage := range s.stages { + if stage.ID == id { + s.currentStage = uint(i) + return nil + } + } + + return ErrStageNotFound +} + +func (s *StagedStreamSync) StageState(stage SyncStageID, tx kv.Tx, db kv.RwDB) (*StageState, error) { + var blockNum uint64 + var err error + if errV := CreateView(context.Background(), db, tx, func(rtx kv.Tx) error { + blockNum, err = GetStageProgress(rtx, stage, s.isBeacon) + if err != nil { + return err + } + return nil + }); errV != nil { + return nil, errV + } + + return &StageState{s, stage, blockNum}, nil +} + +func (s *StagedStreamSync) cleanUp(fromStage int, db kv.RwDB, tx kv.RwTx, firstCycle bool) error { + found := false + for i := 0; i < len(s.pruningOrder); i++ { + if s.pruningOrder[i].ID == s.stages[fromStage].ID { + found = true + } + if !found || s.pruningOrder[i] == nil || s.pruningOrder[i].Disabled { + continue + } + if err := s.pruneStage(firstCycle, s.pruningOrder[i], db, tx); err != nil { + panic(err) + } + } + return nil +} + +func New(ctx context.Context, + bc core.BlockChain, + db kv.RwDB, + stagesList []*Stage, + isBeacon bool, + protocol syncProtocol, + useMemDB bool, + config Config, + logger zerolog.Logger, +) *StagedStreamSync { + + revertStages := make([]*Stage, len(stagesList)) + for i, stageIndex := range DefaultRevertOrder { + for _, s := range stagesList { + if s.ID == stageIndex { + revertStages[i] = s + break + } + } + } + pruneStages := make([]*Stage, len(stagesList)) + for i, stageIndex := range DefaultCleanUpOrder { + for _, s := range stagesList { + if s.ID == stageIndex { + pruneStages[i] = s + break + } + } + } + + logPrefixes := make([]string, len(stagesList)) + for i := range stagesList { + logPrefixes[i] = fmt.Sprintf("%d/%d %s", i+1, len(stagesList), stagesList[i].ID) + } + + status := newStatus() + + return &StagedStreamSync{ + ctx: ctx, + bc: bc, + isBeacon: isBeacon, + db: db, + protocol: protocol, + gbm: nil, + status: status, + inserted: 0, + config: config, + logger: logger, + stages: stagesList, + currentStage: 0, + revertOrder: revertStages, + pruningOrder: pruneStages, + logPrefixes: logPrefixes, + UseMemDB: useMemDB, + } +} + +func (s *StagedStreamSync) doGetCurrentNumberRequest() (uint64, sttypes.StreamID, error) { + ctx, cancel := context.WithTimeout(s.ctx, 10*time.Second) + defer cancel() + + bn, stid, err := s.protocol.GetCurrentBlockNumber(ctx, syncproto.WithHighPriority()) + if err != nil { + return 0, stid, err + } + return bn, stid, nil +} + +func (s *StagedStreamSync) promLabels() prometheus.Labels { + sid := s.bc.ShardID() + return prometheus.Labels{"ShardID": fmt.Sprintf("%d", sid)} +} + +func (s *StagedStreamSync) checkHaveEnoughStreams() error { + numStreams := s.protocol.NumStreams() + if numStreams < s.config.MinStreams { + return fmt.Errorf("number of streams smaller than minimum: %v < %v", + numStreams, s.config.MinStreams) + } + return nil +} + +func (s *StagedStreamSync) SetNewContext(ctx context.Context) error { + for _, s := range s.stages { + s.Handler.SetStageContext(ctx) + } + return nil +} + +func (s *StagedStreamSync) Run(db kv.RwDB, tx kv.RwTx, firstCycle bool) error { + s.prevRevertPoint = nil + s.timings = s.timings[:0] + + for !s.IsDone() { + if s.revertPoint != nil { + s.prevRevertPoint = s.revertPoint + s.revertPoint = nil + if !s.invalidBlock.Active { + for j := 0; j < len(s.revertOrder); j++ { + if s.revertOrder[j] == nil || s.revertOrder[j].Disabled { + continue + } + if err := s.revertStage(firstCycle, s.revertOrder[j], db, tx); err != nil { + utils.Logger().Error(). + Err(err). + Interface("stage id", s.revertOrder[j].ID). + Msgf(WrapStagedSyncMsg("revert stage failed")) + return err + } + } + } + if err := s.SetCurrentStage(s.stages[0].ID); err != nil { + return err + } + firstCycle = false + } + + stage := s.stages[s.currentStage] + + if stage.Disabled { + utils.Logger().Trace(). + Msgf(WrapStagedSyncMsg(fmt.Sprintf("%s disabled. %s", stage.ID, stage.DisabledDescription))) + + s.NextStage() + continue + } + + if err := s.runStage(stage, db, tx, firstCycle, s.invalidBlock.Active); err != nil { + utils.Logger().Error(). + Err(err). + Interface("stage id", stage.ID). + Msgf(WrapStagedSyncMsg("stage failed")) + return err + } + s.NextStage() + } + + if err := s.cleanUp(0, db, tx, firstCycle); err != nil { + utils.Logger().Error(). + Err(err). + Msgf(WrapStagedSyncMsg("stages cleanup failed")) + return err + } + if err := s.SetCurrentStage(s.stages[0].ID); err != nil { + return err + } + if err := printLogs(tx, s.timings); err != nil { + return err + } + s.currentStage = 0 + return nil +} + +func CreateView(ctx context.Context, db kv.RwDB, tx kv.Tx, f func(tx kv.Tx) error) error { + if tx != nil { + return f(tx) + } + return db.View(context.Background(), func(etx kv.Tx) error { + return f(etx) + }) +} + +func ByteCount(b uint64) string { + const unit = 1024 + if b < unit { + return fmt.Sprintf("%dB", b) + } + div, exp := uint64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f%cB", + float64(b)/float64(div), "KMGTPE"[exp]) +} + +func printLogs(tx kv.RwTx, timings []Timing) error { + var logCtx []interface{} + count := 0 + for i := range timings { + if timings[i].took < 50*time.Millisecond { + continue + } + count++ + if count == 50 { + break + } + if timings[i].isRevert { + logCtx = append(logCtx, "Revert "+string(timings[i].stage), timings[i].took.Truncate(time.Millisecond).String()) + } else if timings[i].isCleanUp { + logCtx = append(logCtx, "CleanUp "+string(timings[i].stage), timings[i].took.Truncate(time.Millisecond).String()) + } else { + logCtx = append(logCtx, string(timings[i].stage), timings[i].took.Truncate(time.Millisecond).String()) + } + } + if len(logCtx) > 0 { + utils.Logger().Info(). + Msgf(WrapStagedSyncMsg(fmt.Sprintf("Timings (slower than 50ms) %v", logCtx...))) + } + + if tx == nil { + return nil + } + + if len(logCtx) > 0 { // also don't print this logs if everything is fast + buckets := Buckets + bucketSizes := make([]interface{}, 0, 2*len(buckets)) + for _, bucket := range buckets { + sz, err1 := tx.BucketSize(bucket) + if err1 != nil { + return err1 + } + bucketSizes = append(bucketSizes, bucket, ByteCount(sz)) + } + utils.Logger().Info(). + Msgf(WrapStagedSyncMsg(fmt.Sprintf("Tables %v", bucketSizes...))) + } + tx.CollectMetrics() + return nil +} + +func (s *StagedStreamSync) runStage(stage *Stage, db kv.RwDB, tx kv.RwTx, firstCycle bool, invalidBlockRevert bool) (err error) { + start := time.Now() + stageState, err := s.StageState(stage.ID, tx, db) + if err != nil { + return err + } + + if err = stage.Handler.Exec(firstCycle, invalidBlockRevert, stageState, s, tx); err != nil { + utils.Logger().Error(). + Err(err). + Interface("stage id", stage.ID). + Msgf(WrapStagedSyncMsg("stage failed")) + return fmt.Errorf("[%s] %w", s.LogPrefix(), err) + } + + took := time.Since(start) + if took > 60*time.Second { + logPrefix := s.LogPrefix() + utils.Logger().Info(). + Msgf(WrapStagedSyncMsg(fmt.Sprintf("%s: DONE in %d", logPrefix, took))) + + } + s.timings = append(s.timings, Timing{stage: stage.ID, took: took}) + return nil +} + +func (s *StagedStreamSync) revertStage(firstCycle bool, stage *Stage, db kv.RwDB, tx kv.RwTx) error { + start := time.Now() + stageState, err := s.StageState(stage.ID, tx, db) + if err != nil { + return err + } + + revert := s.NewRevertState(stage.ID, *s.revertPoint) + + if stageState.BlockNumber <= revert.RevertPoint { + return nil + } + + if err = s.SetCurrentStage(stage.ID); err != nil { + return err + } + + err = stage.Handler.Revert(firstCycle, revert, stageState, tx) + if err != nil { + return fmt.Errorf("[%s] %w", s.LogPrefix(), err) + } + + took := time.Since(start) + if took > 60*time.Second { + logPrefix := s.LogPrefix() + utils.Logger().Info(). + Msgf(WrapStagedSyncMsg(fmt.Sprintf("%s: Revert done in %d", logPrefix, took))) + } + s.timings = append(s.timings, Timing{isRevert: true, stage: stage.ID, took: took}) + return nil +} + +func (s *StagedStreamSync) pruneStage(firstCycle bool, stage *Stage, db kv.RwDB, tx kv.RwTx) error { + start := time.Now() + + stageState, err := s.StageState(stage.ID, tx, db) + if err != nil { + return err + } + + prune, err := s.CleanUpStageState(stage.ID, stageState.BlockNumber, tx, db) + if err != nil { + return err + } + if err = s.SetCurrentStage(stage.ID); err != nil { + return err + } + + err = stage.Handler.CleanUp(firstCycle, prune, tx) + if err != nil { + return fmt.Errorf("[%s] %w", s.LogPrefix(), err) + } + + took := time.Since(start) + if took > 60*time.Second { + logPrefix := s.LogPrefix() + utils.Logger().Info(). + Msgf(WrapStagedSyncMsg(fmt.Sprintf("%s: CleanUp done in %d", logPrefix, took))) + } + s.timings = append(s.timings, Timing{isCleanUp: true, stage: stage.ID, took: took}) + return nil +} + +// DisableAllStages disables all stages including their reverts +func (s *StagedStreamSync) DisableAllStages() []SyncStageID { + var backupEnabledIds []SyncStageID + for i := range s.stages { + if !s.stages[i].Disabled { + backupEnabledIds = append(backupEnabledIds, s.stages[i].ID) + } + } + for i := range s.stages { + s.stages[i].Disabled = true + } + return backupEnabledIds +} + +func (s *StagedStreamSync) DisableStages(ids ...SyncStageID) { + for i := range s.stages { + for _, id := range ids { + if s.stages[i].ID != id { + continue + } + s.stages[i].Disabled = true + } + } +} + +func (s *StagedStreamSync) EnableStages(ids ...SyncStageID) { + for i := range s.stages { + for _, id := range ids { + if s.stages[i].ID != id { + continue + } + s.stages[i].Disabled = false + } + } +} diff --git a/api/service/stagedstreamsync/stages.go b/api/service/stagedstreamsync/stages.go new file mode 100644 index 0000000000..dc607416b8 --- /dev/null +++ b/api/service/stagedstreamsync/stages.go @@ -0,0 +1,71 @@ +package stagedstreamsync + +import ( + "github.com/ledgerwatch/erigon-lib/kv" +) + +// SyncStageID represents the stages in the Mode.StagedSync mode +type SyncStageID string + +const ( + Heads SyncStageID = "Heads" // Heads are downloaded + ShortRange SyncStageID = "ShortRange" // short range + SyncEpoch SyncStageID = "SyncEpoch" // epoch sync + BlockBodies SyncStageID = "BlockBodies" // Block bodies are downloaded, TxHash and UncleHash are getting verified + States SyncStageID = "States" // will construct most recent state from downloaded blocks + Finish SyncStageID = "Finish" // Nominal stage after all other stages +) + +func GetStageName(stage string, isBeacon bool, prune bool) string { + name := stage + if isBeacon { + name = "beacon_" + name + } + if prune { + name = "prune_" + name + } + return name +} + +func GetStageID(stage SyncStageID, isBeacon bool, prune bool) []byte { + return []byte(GetStageName(string(stage), isBeacon, prune)) +} + +func GetBucketName(bucketName string, isBeacon bool) string { + name := bucketName + if isBeacon { + name = "Beacon" + name + } + return name +} + +// GetStageProgress retrieves saved progress of a given sync stage from the database +func GetStageProgress(db kv.Getter, stage SyncStageID, isBeacon bool) (uint64, error) { + stgID := GetStageID(stage, isBeacon, false) + v, err := db.GetOne(kv.SyncStageProgress, stgID) + if err != nil { + return 0, err + } + return unmarshalData(v) +} + +// SaveStageProgress saves progress of given sync stage +func SaveStageProgress(db kv.Putter, stage SyncStageID, isBeacon bool, progress uint64) error { + stgID := GetStageID(stage, isBeacon, false) + return db.Put(kv.SyncStageProgress, stgID, marshalData(progress)) +} + +// GetStageCleanUpProgress retrieves saved progress of given sync stage from the database +func GetStageCleanUpProgress(db kv.Getter, stage SyncStageID, isBeacon bool) (uint64, error) { + stgID := GetStageID(stage, isBeacon, true) + v, err := db.GetOne(kv.SyncStageProgress, stgID) + if err != nil { + return 0, err + } + return unmarshalData(v) +} + +func SaveStageCleanUpProgress(db kv.Putter, stage SyncStageID, isBeacon bool, progress uint64) error { + stgID := GetStageID(stage, isBeacon, true) + return db.Put(kv.SyncStageProgress, stgID, marshalData(progress)) +} diff --git a/api/service/stagedstreamsync/syncing.go b/api/service/stagedstreamsync/syncing.go new file mode 100644 index 0000000000..676eb3551e --- /dev/null +++ b/api/service/stagedstreamsync/syncing.go @@ -0,0 +1,320 @@ +package stagedstreamsync + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/internal/utils" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/harmony-one/harmony/shard" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/mdbx" + "github.com/ledgerwatch/erigon-lib/kv/memdb" + "github.com/ledgerwatch/log/v3" + "github.com/pkg/errors" + "github.com/rs/zerolog" +) + +const ( + BlocksBucket = "BlockBodies" + BlockSignaturesBucket = "BlockSignatures" + StageProgressBucket = "StageProgress" + + // cache db keys + LastBlockHeight = "LastBlockHeight" + LastBlockHash = "LastBlockHash" +) + +var Buckets = []string{ + BlocksBucket, + BlockSignaturesBucket, + StageProgressBucket, +} + +// CreateStagedSync creates an instance of staged sync +func CreateStagedSync(ctx context.Context, + bc core.BlockChain, + UseMemDB bool, + protocol syncProtocol, + config Config, + logger zerolog.Logger, + logProgress bool, +) (*StagedStreamSync, error) { + + isBeacon := bc.ShardID() == shard.BeaconChainShardID + + var mainDB kv.RwDB + dbs := make([]kv.RwDB, config.Concurrency) + if UseMemDB { + mainDB = memdb.New() + for i := 0; i < config.Concurrency; i++ { + dbs[i] = memdb.New() + } + } else { + mainDB = mdbx.NewMDBX(log.New()).Path(GetBlockDbPath(isBeacon, -1)).MustOpen() + for i := 0; i < config.Concurrency; i++ { + dbPath := GetBlockDbPath(isBeacon, i) + dbs[i] = mdbx.NewMDBX(log.New()).Path(dbPath).MustOpen() + } + } + + if errInitDB := initDB(ctx, mainDB, dbs, config.Concurrency); errInitDB != nil { + return nil, errInitDB + } + + stageHeadsCfg := NewStageHeadersCfg(ctx, bc, mainDB) + stageShortRangeCfg := NewStageShortRangeCfg(ctx, bc, mainDB) + stageSyncEpochCfg := NewStageEpochCfg(ctx, bc, mainDB) + stageBodiesCfg := NewStageBodiesCfg(ctx, bc, mainDB, dbs, config.Concurrency, protocol, isBeacon, logProgress) + stageStatesCfg := NewStageStatesCfg(ctx, bc, mainDB, dbs, config.Concurrency, logger, logProgress) + stageFinishCfg := NewStageFinishCfg(ctx, mainDB) + + stages := DefaultStages(ctx, + stageHeadsCfg, + stageSyncEpochCfg, + stageShortRangeCfg, + stageBodiesCfg, + stageStatesCfg, + stageFinishCfg, + ) + + return New(ctx, + bc, + mainDB, + stages, + isBeacon, + protocol, + UseMemDB, + config, + logger, + ), nil +} + +// initDB inits the sync loop main database and create buckets +func initDB(ctx context.Context, mainDB kv.RwDB, dbs []kv.RwDB, concurrency int) error { + + // create buckets for mainDB + tx, errRW := mainDB.BeginRw(ctx) + if errRW != nil { + return errRW + } + defer tx.Rollback() + + for _, name := range Buckets { + if err := tx.CreateBucket(GetStageName(name, false, false)); err != nil { + return err + } + } + if err := tx.Commit(); err != nil { + return err + } + + // create buckets for block cache DBs + for _, db := range dbs { + tx, errRW := db.BeginRw(ctx) + if errRW != nil { + return errRW + } + + if err := tx.CreateBucket(BlocksBucket); err != nil { + return err + } + if err := tx.CreateBucket(BlockSignaturesBucket); err != nil { + return err + } + + if err := tx.Commit(); err != nil { + return err + } + } + + return nil +} + +func GetBlockDbPath(beacon bool, loopID int) string { + if beacon { + if loopID >= 0 { + return fmt.Sprintf("%s_%d", "cache/beacon_blocks_db", loopID) + } else { + return "cache/beacon_blocks_db_main" + } + } else { + if loopID >= 0 { + return fmt.Sprintf("%s_%d", "cache/blocks_db", loopID) + } else { + return "cache/blocks_db_main" + } + } +} + +// doSync does the long range sync. +// One LongRangeSync consists of several iterations. +// For each iteration, estimate the current block number, then fetch block & insert to blockchain +func (s *StagedStreamSync) doSync(downloaderContext context.Context, initSync bool) (int, error) { + + var totalInserted int + + s.initSync = initSync + + if err := s.checkPrerequisites(); err != nil { + return 0, err + } + + var estimatedHeight uint64 + if initSync { + if h, err := s.estimateCurrentNumber(); err != nil { + return 0, err + } else { + estimatedHeight = h + //TODO: use directly currentCycle var + s.status.setTargetBN(estimatedHeight) + } + if curBN := s.bc.CurrentBlock().NumberU64(); estimatedHeight <= curBN { + s.logger.Info().Uint64("current number", curBN).Uint64("target number", estimatedHeight). + Msg(WrapStagedSyncMsg("early return of long range sync")) + return 0, nil + } + + s.startSyncing() + defer s.finishSyncing() + } + + for { + ctx, cancel := context.WithCancel(downloaderContext) + s.ctx = ctx + s.SetNewContext(ctx) + + n, err := s.doSyncCycle(ctx, initSync) + if err != nil { + pl := s.promLabels() + pl["error"] = err.Error() + numFailedDownloadCounterVec.With(pl).Inc() + + cancel() + return totalInserted + n, err + } + cancel() + + totalInserted += n + + // if it's not long range sync, skip loop + if n < LastMileBlocksThreshold || !initSync { + return totalInserted, nil + } + } + +} + +func (s *StagedStreamSync) doSyncCycle(ctx context.Context, initSync bool) (int, error) { + + // TODO: initSync=true means currentCycleNumber==0, so we can remove initSync + + var totalInserted int + + s.inserted = 0 + startHead := s.bc.CurrentBlock().NumberU64() + canRunCycleInOneTransaction := false + + var tx kv.RwTx + if canRunCycleInOneTransaction { + var err error + if tx, err = s.DB().BeginRw(context.Background()); err != nil { + return totalInserted, err + } + defer tx.Rollback() + } + + startTime := time.Now() + + // Do one cycle of staged sync + initialCycle := s.currentCycle.Number == 0 + if err := s.Run(s.DB(), tx, initialCycle); err != nil { + utils.Logger().Error(). + Err(err). + Bool("isBeacon", s.isBeacon). + Uint32("shard", s.bc.ShardID()). + Uint64("currentHeight", startHead). + Msgf(WrapStagedSyncMsg("sync cycle failed")) + return totalInserted, err + } + + totalInserted += s.inserted + + s.currentCycle.lock.Lock() + s.currentCycle.Number++ + s.currentCycle.lock.Unlock() + + // calculating sync speed (blocks/second) + if s.LogProgress && s.inserted > 0 { + dt := time.Now().Sub(startTime).Seconds() + speed := float64(0) + if dt > 0 { + speed = float64(s.inserted) / dt + } + syncSpeed := fmt.Sprintf("%.2f", speed) + fmt.Println("sync speed:", syncSpeed, "blocks/s") + } + + return totalInserted, nil +} + +func (s *StagedStreamSync) startSyncing() { + s.status.startSyncing() + if s.evtDownloadStartedSubscribed { + s.evtDownloadStarted.Send(struct{}{}) + } +} + +func (s *StagedStreamSync) finishSyncing() { + s.status.finishSyncing() + if s.evtDownloadFinishedSubscribed { + s.evtDownloadFinished.Send(struct{}{}) + } +} + +func (s *StagedStreamSync) checkPrerequisites() error { + return s.checkHaveEnoughStreams() +} + +// estimateCurrentNumber roughly estimates the current block number. +// The block number does not need to be exact, but just a temporary target of the iteration +func (s *StagedStreamSync) estimateCurrentNumber() (uint64, error) { + var ( + cnResults = make(map[sttypes.StreamID]uint64) + lock sync.Mutex + wg sync.WaitGroup + ) + wg.Add(s.config.Concurrency) + for i := 0; i != s.config.Concurrency; i++ { + go func() { + defer wg.Done() + bn, stid, err := s.doGetCurrentNumberRequest() + if err != nil { + s.logger.Err(err).Str("streamID", string(stid)). + Msg(WrapStagedSyncMsg("getCurrentNumber request failed")) + if !errors.Is(err, context.Canceled) { + s.protocol.StreamFailed(stid, "getCurrentNumber request failed") + } + return + } + lock.Lock() + cnResults[stid] = bn + lock.Unlock() + }() + } + wg.Wait() + + if len(cnResults) == 0 { + select { + case <-s.ctx.Done(): + return 0, s.ctx.Err() + default: + } + return 0, errors.New("zero block number response from remote nodes") + } + bn := computeBlockNumberByMaxVote(cnResults) + return bn, nil +} diff --git a/api/service/stagedstreamsync/types.go b/api/service/stagedstreamsync/types.go new file mode 100644 index 0000000000..00cfba79a2 --- /dev/null +++ b/api/service/stagedstreamsync/types.go @@ -0,0 +1,287 @@ +package stagedstreamsync + +import ( + "container/heap" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/harmony/core/types" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" +) + +var ( + emptyHash common.Hash +) + +type status struct { + isSyncing bool + targetBN uint64 + lock sync.Mutex +} + +func newStatus() status { + return status{} +} + +func (s *status) startSyncing() { + s.lock.Lock() + defer s.lock.Unlock() + + s.isSyncing = true +} + +func (s *status) setTargetBN(val uint64) { + s.lock.Lock() + defer s.lock.Unlock() + + s.targetBN = val +} + +func (s *status) finishSyncing() { + s.lock.Lock() + defer s.lock.Unlock() + + s.isSyncing = false + s.targetBN = 0 +} + +func (s *status) get() (bool, uint64) { + s.lock.Lock() + defer s.lock.Unlock() + + return s.isSyncing, s.targetBN +} + +type getBlocksResult struct { + bns []uint64 + blocks []*types.Block + stid sttypes.StreamID +} + +type resultQueue struct { + results *priorityQueue + lock sync.Mutex +} + +func newResultQueue() *resultQueue { + pq := make(priorityQueue, 0, 200) // 200 - rough estimate + heap.Init(&pq) + return &resultQueue{ + results: &pq, + } +} + +// addBlockResults adds the blocks to the result queue to be processed by insertChainLoop. +// If a nil block is detected in the block list, will not process further blocks. +func (rq *resultQueue) addBlockResults(blocks []*types.Block, stid sttypes.StreamID) { + rq.lock.Lock() + defer rq.lock.Unlock() + + for _, block := range blocks { + if block == nil { + continue + } + heap.Push(rq.results, &blockResult{ + block: block, + stid: stid, + }) + } + return +} + +// popBlockResults pop a continuous list of blocks starting at expStartBN with capped size. +// Return the stale block numbers as the second return value +func (rq *resultQueue) popBlockResults(expStartBN uint64, cap int) ([]*blockResult, []uint64) { + rq.lock.Lock() + defer rq.lock.Unlock() + + var ( + res = make([]*blockResult, 0, cap) + stales []uint64 + ) + + for cnt := 0; rq.results.Len() > 0 && cnt < cap; cnt++ { + br := heap.Pop(rq.results).(*blockResult) + // stale block number + if br.block.NumberU64() < expStartBN { + stales = append(stales, br.block.NumberU64()) + continue + } + if br.block.NumberU64() != expStartBN { + heap.Push(rq.results, br) + return res, stales + } + res = append(res, br) + expStartBN++ + } + return res, stales +} + +// removeResultsByStreamID removes the block results of the given stream, returns the block +// number removed from the queue +func (rq *resultQueue) removeResultsByStreamID(stid sttypes.StreamID) []uint64 { + rq.lock.Lock() + defer rq.lock.Unlock() + + var removed []uint64 + +Loop: + for { + for i, res := range *rq.results { + blockRes := res.(*blockResult) + if blockRes.stid == stid { + rq.removeByIndex(i) + removed = append(removed, blockRes.block.NumberU64()) + goto Loop + } + } + break + } + return removed +} + +func (rq *resultQueue) length() int { + return len(*rq.results) +} + +func (rq *resultQueue) removeByIndex(index int) { + heap.Remove(rq.results, index) +} + +// bnPrioritizedItem is the item which uses block number to determine its priority +type bnPrioritizedItem interface { + getBlockNumber() uint64 +} + +type blockResult struct { + block *types.Block + stid sttypes.StreamID +} + +func (br *blockResult) getBlockNumber() uint64 { + return br.block.NumberU64() +} + +func blockResultsToBlocks(results []*blockResult) []*types.Block { + blocks := make([]*types.Block, 0, len(results)) + + for _, result := range results { + blocks = append(blocks, result.block) + } + return blocks +} + +type ( + prioritizedNumber uint64 + + prioritizedNumbers struct { + q *priorityQueue + } +) + +func (b prioritizedNumber) getBlockNumber() uint64 { + return uint64(b) +} + +func newPrioritizedNumbers() *prioritizedNumbers { + pqs := make(priorityQueue, 0) + heap.Init(&pqs) + return &prioritizedNumbers{ + q: &pqs, + } +} + +func (pbs *prioritizedNumbers) push(bn uint64) { + heap.Push(pbs.q, prioritizedNumber(bn)) +} + +func (pbs *prioritizedNumbers) pop() uint64 { + if pbs.q.Len() == 0 { + return 0 + } + item := heap.Pop(pbs.q) + return uint64(item.(prioritizedNumber)) +} + +func (pbs *prioritizedNumbers) length() int { + return len(*pbs.q) +} + +type ( + blockByNumber types.Block + + // blocksByNumber is the priority queue ordered by number + blocksByNumber struct { + q *priorityQueue + cap int + } +) + +func (b *blockByNumber) getBlockNumber() uint64 { + raw := (*types.Block)(b) + return raw.NumberU64() +} + +func newBlocksByNumber(cap int) *blocksByNumber { + pqs := make(priorityQueue, 0) + heap.Init(&pqs) + return &blocksByNumber{ + q: &pqs, + cap: cap, + } +} + +func (bs *blocksByNumber) push(b *types.Block) { + heap.Push(bs.q, (*blockByNumber)(b)) + for bs.q.Len() > bs.cap { + heap.Pop(bs.q) + } +} + +func (bs *blocksByNumber) pop() *types.Block { + if bs.q.Len() == 0 { + return nil + } + item := heap.Pop(bs.q) + return (*types.Block)(item.(*blockByNumber)) +} + +func (bs *blocksByNumber) len() int { + return bs.q.Len() +} + +// priorityQueue is a priority queue with lowest block number with highest priority +type priorityQueue []bnPrioritizedItem + +func (q priorityQueue) Len() int { + return len(q) +} + +func (q priorityQueue) Less(i, j int) bool { + bn1 := q[i].getBlockNumber() + bn2 := q[j].getBlockNumber() + return bn1 < bn2 // small block number has higher priority +} + +func (q priorityQueue) Swap(i, j int) { + q[i], q[j] = q[j], q[i] +} + +func (q *priorityQueue) Push(x interface{}) { + item, ok := x.(bnPrioritizedItem) + if !ok { + panic("wrong type of getBlockNumber interface") + } + *q = append(*q, item) +} + +func (q *priorityQueue) Pop() interface{} { + prev := *q + n := len(prev) + if n == 0 { + return nil + } + res := prev[n-1] + *q = prev[0 : n-1] + return res +} diff --git a/api/service/stagedstreamsync/types_test.go b/api/service/stagedstreamsync/types_test.go new file mode 100644 index 0000000000..1890608b25 --- /dev/null +++ b/api/service/stagedstreamsync/types_test.go @@ -0,0 +1,266 @@ +package stagedstreamsync + +import ( + "container/heap" + "fmt" + "math/big" + "strings" + "testing" + + "github.com/harmony-one/harmony/block" + headerV3 "github.com/harmony-one/harmony/block/v3" + "github.com/harmony-one/harmony/core/types" + bls_cosi "github.com/harmony-one/harmony/crypto/bls" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" +) + +func TestResultQueue_AddBlockResults(t *testing.T) { + tests := []struct { + initBNs []uint64 + addBNs []uint64 + expSize int + }{ + { + initBNs: []uint64{}, + addBNs: []uint64{1, 2, 3, 4}, + expSize: 4, + }, + { + initBNs: []uint64{1, 2, 3, 4}, + addBNs: []uint64{5, 6, 7, 8}, + expSize: 8, + }, + } + for i, test := range tests { + rq := makeTestResultQueue(test.initBNs) + rq.addBlockResults(makeTestBlocks(test.addBNs), "") + + if rq.results.Len() != test.expSize { + t.Errorf("Test %v: unexpected size: %v / %v", i, rq.results.Len(), test.expSize) + } + } +} + +func TestResultQueue_PopBlockResults(t *testing.T) { + tests := []struct { + initBNs []uint64 + cap int + expStart uint64 + expSize int + staleSize int + }{ + { + initBNs: []uint64{1, 2, 3, 4, 5}, + cap: 3, + expStart: 1, + expSize: 3, + staleSize: 0, + }, + { + initBNs: []uint64{1, 2, 3, 4, 5}, + cap: 10, + expStart: 1, + expSize: 5, + staleSize: 0, + }, + { + initBNs: []uint64{1, 3, 4, 5}, + cap: 10, + expStart: 1, + expSize: 1, + staleSize: 0, + }, + { + initBNs: []uint64{1, 2, 3, 4, 5}, + cap: 10, + expStart: 0, + expSize: 0, + staleSize: 0, + }, + { + initBNs: []uint64{1, 1, 1, 1, 2}, + cap: 10, + expStart: 1, + expSize: 2, + staleSize: 3, + }, + { + initBNs: []uint64{1, 2, 3, 4, 5}, + cap: 10, + expStart: 2, + expSize: 4, + staleSize: 1, + }, + } + for i, test := range tests { + rq := makeTestResultQueue(test.initBNs) + res, stales := rq.popBlockResults(test.expStart, test.cap) + if len(res) != test.expSize { + t.Errorf("Test %v: unexpect size %v / %v", i, len(res), test.expSize) + } + if len(stales) != test.staleSize { + t.Errorf("Test %v: unexpect stale size %v / %v", i, len(stales), test.staleSize) + } + } +} + +func TestResultQueue_RemoveResultsByStreamID(t *testing.T) { + tests := []struct { + rq *resultQueue + rmStreamID sttypes.StreamID + removed int + expSize int + }{ + { + rq: makeTestResultQueue([]uint64{1, 2, 3, 4}), + rmStreamID: "test stream id", + removed: 4, + expSize: 0, + }, + { + rq: func() *resultQueue { + rq := makeTestResultQueue([]uint64{2, 3, 4, 5}) + rq.addBlockResults([]*types.Block{ + makeTestBlock(1), + makeTestBlock(5), + makeTestBlock(6), + }, "another test stream id") + return rq + }(), + rmStreamID: "test stream id", + removed: 4, + expSize: 3, + }, + { + rq: func() *resultQueue { + rq := makeTestResultQueue([]uint64{2, 3, 4, 5}) + rq.addBlockResults([]*types.Block{ + makeTestBlock(1), + makeTestBlock(5), + makeTestBlock(6), + }, "another test stream id") + return rq + }(), + rmStreamID: "another test stream id", + removed: 3, + expSize: 4, + }, + } + for i, test := range tests { + res := test.rq.removeResultsByStreamID(test.rmStreamID) + if len(res) != test.removed { + t.Errorf("Test %v: unexpected number removed %v / %v", i, len(res), test.removed) + } + if gotSize := test.rq.results.Len(); gotSize != test.expSize { + t.Errorf("Test %v: unexpected number after removal %v / %v", i, gotSize, test.expSize) + } + } +} + +func makeTestResultQueue(bns []uint64) *resultQueue { + rq := newResultQueue() + for _, bn := range bns { + heap.Push(rq.results, &blockResult{ + block: makeTestBlock(bn), + stid: "test stream id", + }) + } + return rq +} + +func TestPrioritizedBlocks(t *testing.T) { + addBNs := []uint64{4, 7, 6, 9} + + bns := newPrioritizedNumbers() + for _, bn := range addBNs { + bns.push(bn) + } + prevBN := uint64(0) + for len(*bns.q) > 0 { + b := bns.pop() + if b < prevBN { + t.Errorf("number not incrementing") + } + prevBN = b + } + if last := bns.pop(); last != 0 { + t.Errorf("last elem is not 0") + } +} + +func TestBlocksByNumber(t *testing.T) { + addBNs := []uint64{4, 7, 6, 9} + + bns := newBlocksByNumber(10) + for _, bn := range addBNs { + bns.push(makeTestBlock(bn)) + } + if bns.len() != len(addBNs) { + t.Errorf("size unexpected: %v / %v", bns.len(), len(addBNs)) + } + prevBN := uint64(0) + for len(*bns.q) > 0 { + b := bns.pop() + if b.NumberU64() < prevBN { + t.Errorf("number not incrementing") + } + prevBN = b.NumberU64() + } + if lastBlock := bns.pop(); lastBlock != nil { + t.Errorf("last block is not nil") + } +} + +func TestPriorityQueue(t *testing.T) { + testBNs := []uint64{1, 9, 2, 4, 5, 12} + pq := make(priorityQueue, 0, 10) + heap.Init(&pq) + for _, bn := range testBNs { + heap.Push(&pq, &blockResult{ + block: makeTestBlock(bn), + stid: "", + }) + } + cmpBN := uint64(0) + for pq.Len() > 0 { + bn := heap.Pop(&pq).(*blockResult).block.NumberU64() + if bn < cmpBN { + t.Errorf("not incrementing") + } + cmpBN = bn + } + if pq.Len() != 0 { + t.Errorf("after poping, size not 0") + } +} + +func makeTestBlocks(bns []uint64) []*types.Block { + blocks := make([]*types.Block, 0, len(bns)) + for _, bn := range bns { + blocks = append(blocks, makeTestBlock(bn)) + } + return blocks +} + +func makeTestBlock(bn uint64) *types.Block { + testHeader := &block.Header{Header: headerV3.NewHeader()} + testHeader.SetNumber(big.NewInt(int64(bn))) + testHeader.SetLastCommitSignature(bls_cosi.SerializedSignature{}) + testHeader.SetLastCommitBitmap(make([]byte, 10)) + block := types.NewBlockWithHeader(testHeader) + block.SetCurrentCommitSig(make([]byte, 106)) + return block +} + +func assertError(got, expect error) error { + if (got == nil) != (expect == nil) { + return fmt.Errorf("unexpected error [%v] / [%v]", got, expect) + } + if (got == nil) || (expect == nil) { + return nil + } + if !strings.Contains(got.Error(), expect.Error()) { + return fmt.Errorf("unexpected error [%v] / [%v]", got, expect) + } + return nil +} diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 437a91dc59..6c83636673 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -39,6 +39,7 @@ import ( "github.com/harmony-one/harmony/api/service" "github.com/harmony-one/harmony/api/service/pprof" "github.com/harmony-one/harmony/api/service/prometheus" + "github.com/harmony-one/harmony/api/service/stagedstreamsync" "github.com/harmony-one/harmony/api/service/synchronize" "github.com/harmony-one/harmony/common/fdlimit" "github.com/harmony-one/harmony/common/ntp" @@ -415,7 +416,11 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { // Setup services if hc.Sync.Enabled { - setupSyncService(currentNode, myHost, hc) + if hc.Sync.StagedSync { + setupStagedSyncService(currentNode, myHost, hc) + } else { + setupSyncService(currentNode, myHost, hc) + } } if currentNode.NodeConfig.Role() == nodeconfig.Validator { currentNode.RegisterValidatorServices() @@ -906,6 +911,45 @@ func setupSyncService(node *node.Node, host p2p.Host, hc harmonyconfig.HarmonyCo } } +func setupStagedSyncService(node *node.Node, host p2p.Host, hc harmonyconfig.HarmonyConfig) { + blockchains := []core.BlockChain{node.Blockchain()} + if !node.IsRunningBeaconChain() { + blockchains = append(blockchains, node.Beaconchain()) + } + + sConfig := stagedstreamsync.Config{ + ServerOnly: !hc.Sync.Downloader, + Network: nodeconfig.NetworkType(hc.Network.NetworkType), + Concurrency: hc.Sync.Concurrency, + MinStreams: hc.Sync.MinPeers, + InitStreams: hc.Sync.InitStreams, + SmSoftLowCap: hc.Sync.DiscSoftLowCap, + SmHardLowCap: hc.Sync.DiscHardLowCap, + SmHiCap: hc.Sync.DiscHighCap, + SmDiscBatch: hc.Sync.DiscBatch, + LogProgress: node.NodeConfig.LogProgress, + } + + // If we are running side chain, we will need to do some extra works for beacon + // sync. + if !node.IsRunningBeaconChain() { + sConfig.BHConfig = &stagedstreamsync.BeaconHelperConfig{ + BlockC: node.BeaconBlockChannel, + InsertHook: node.BeaconSyncHook, + } + } + + //Setup stream sync service + s := stagedstreamsync.NewService(host, blockchains, sConfig) + + node.RegisterService(service.StagedStreamSync, s) + + d := s.Downloaders.GetShardDownloader(node.Blockchain().ShardID()) + if hc.Sync.Downloader && hc.General.NodeType != nodeTypeExplorer { + node.Consensus.SetDownloader(d) // Set downloader when stream client is active + } +} + func setupBlacklist(hc harmonyconfig.HarmonyConfig) (map[ethCommon.Address]struct{}, error) { rosetta_common.InitRosettaFile(hc.TxPool.RosettaFixFile) diff --git a/go.mod b/go.mod index d45f63e26f..883a62f3fe 100644 --- a/go.mod +++ b/go.mod @@ -165,6 +165,7 @@ require ( github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect + github.com/libp2p/go-libp2p-core v0.20.1 // indirect github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect github.com/libp2p/go-msgio v0.2.0 // indirect diff --git a/go.sum b/go.sum index 17632fa25d..0bcfa714fb 100644 --- a/go.sum +++ b/go.sum @@ -596,6 +596,8 @@ github.com/libp2p/go-libp2p v0.24.0 h1:DQk/5bBon+yUVIGTeRVBmOYpZzoBHx/VTC0xoLgJG github.com/libp2p/go-libp2p v0.24.0/go.mod h1:28t24CYDlnBs23rIs1OclU89YbhgibrBq2LFbMe+cFw= github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= +github.com/libp2p/go-libp2p-core v0.20.1 h1:fQz4BJyIFmSZAiTbKV8qoYhEH5Dtv/cVhZbG3Ib/+Cw= +github.com/libp2p/go-libp2p-core v0.20.1/go.mod h1:6zR8H7CvQWgYLsbG4on6oLNSGcyKaYFSEYyDt51+bIY= github.com/libp2p/go-libp2p-kad-dht v0.19.0 h1:2HuiInHZTm9ZvQajaqdaPLHr0PCKKigWiflakimttE0= github.com/libp2p/go-libp2p-kad-dht v0.19.0/go.mod h1:qPIXdiZsLczhV4/+4EO1jE8ae0YCW4ZOogc4WVIyTEU= github.com/libp2p/go-libp2p-kbucket v0.5.0 h1:g/7tVm8ACHDxH29BGrpsQlnNeu+6OF1A9bno/4/U1oA= diff --git a/hmy/downloader/const.go b/hmy/downloader/const.go index ac292d9747..a6cafb918b 100644 --- a/hmy/downloader/const.go +++ b/hmy/downloader/const.go @@ -35,7 +35,8 @@ type ( // Only run stream sync protocol as a server. // TODO: remove this when stream sync is fully up. ServerOnly bool - + // use staged sync + Staged bool // parameters Network nodeconfig.NetworkType Concurrency int // Number of concurrent sync requests diff --git a/hmy/downloader/metric.go b/hmy/downloader/metric.go index 2995db7b31..9338fd1304 100644 --- a/hmy/downloader/metric.go +++ b/hmy/downloader/metric.go @@ -23,7 +23,7 @@ var ( consensusTriggeredDownloadCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "consensus_trigger", Help: "number of times consensus triggered download task", }, @@ -33,7 +33,7 @@ var ( longRangeSyncedBlockCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "num_blocks_synced_long_range", Help: "number of blocks synced in long range sync", }, @@ -43,7 +43,7 @@ var ( longRangeFailInsertedBlockCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "num_blocks_failed_long_range", Help: "number of blocks failed to insert into change in long range sync", }, @@ -53,7 +53,7 @@ var ( numShortRangeCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "num_short_range", Help: "number of short range sync is triggered", }, @@ -63,7 +63,7 @@ var ( numFailedDownloadCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "failed_download", Help: "number of downloading is failed", }, @@ -73,7 +73,7 @@ var ( numBlocksInsertedShortRangeHistogramVec = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "num_blocks_inserted_short_range", Help: "number of blocks inserted for each short range sync", // Buckets: 0, 1, 2, 4, +INF (capped at 10) @@ -85,7 +85,7 @@ var ( numBlocksInsertedBeaconHelperCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "num_blocks_inserted_beacon_helper", Help: "number of blocks inserted from beacon helper", }, diff --git a/hmy/downloader/shortrange.go b/hmy/downloader/shortrange.go index f1f6e402ce..8276911d4f 100644 --- a/hmy/downloader/shortrange.go +++ b/hmy/downloader/shortrange.go @@ -20,7 +20,7 @@ import ( // doShortRangeSync does the short range sync. // Compared with long range sync, short range sync is more focused on syncing to the latest block. // It consist of 3 steps: -// 1. Obtain the block hashes and ompute the longest hash chain.. +// 1. Obtain the block hashes and compute the longest hash chain.. // 2. Get blocks by hashes from computed hash chain. // 3. Insert the blocks to blockchain. func (d *Downloader) doShortRangeSync() (int, error) { diff --git a/node/node_syncing.go b/node/node_syncing.go index d9a6c95b48..2219be96da 100644 --- a/node/node_syncing.go +++ b/node/node_syncing.go @@ -22,11 +22,11 @@ import ( "github.com/harmony-one/harmony/api/service/legacysync" legdownloader "github.com/harmony-one/harmony/api/service/legacysync/downloader" downloader_pb "github.com/harmony-one/harmony/api/service/legacysync/downloader/proto" + "github.com/harmony-one/harmony/api/service/stagedstreamsync" "github.com/harmony-one/harmony/api/service/stagedsync" "github.com/harmony-one/harmony/api/service/synchronize" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" - "github.com/harmony-one/harmony/hmy/downloader" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/node/worker" @@ -811,7 +811,7 @@ func (node *Node) legacySyncStatus(shardID uint32) (bool, uint64, uint64) { } } -// IsOutOfSync return whether the node is out of sync of the given hsardID +// IsOutOfSync return whether the node is out of sync of the given shardID func (node *Node) IsOutOfSync(shardID uint32) bool { ds := node.getDownloaders() if ds == nil || !ds.IsActive() { @@ -859,14 +859,36 @@ func (node *Node) SyncPeers() map[string]int { return res } -func (node *Node) getDownloaders() *downloader.Downloaders { - syncService := node.serviceManager.GetService(service.Synchronize) - if syncService == nil { - return nil - } - dsService, ok := syncService.(*synchronize.Service) - if !ok { - return nil +type Downloaders interface { + Start() + Close() + DownloadAsync(shardID uint32) + // GetShardDownloader(shardID uint32) *Downloader + NumPeers() map[uint32]int + SyncStatus(shardID uint32) (bool, uint64, uint64) + IsActive() bool +} + +func (node *Node) getDownloaders() Downloaders { + if node.NodeConfig.StagedSync { + syncService := node.serviceManager.GetService(service.StagedStreamSync) + if syncService == nil { + return nil + } + dsService, ok := syncService.(*stagedstreamsync.StagedStreamSyncService) + if !ok { + return nil + } + return dsService.Downloaders + } else { + syncService := node.serviceManager.GetService(service.Synchronize) + if syncService == nil { + return nil + } + dsService, ok := syncService.(*synchronize.Service) + if !ok { + return nil + } + return dsService.Downloaders } - return dsService.Downloaders } diff --git a/p2p/stream/common/requestmanager/interface_test.go b/p2p/stream/common/requestmanager/interface_test.go index fe163164d7..76399e859e 100644 --- a/p2p/stream/common/requestmanager/interface_test.go +++ b/p2p/stream/common/requestmanager/interface_test.go @@ -114,6 +114,18 @@ func (st *testStream) CloseOnExit() error { return nil } +func (st *testStream) FailedTimes() int { + return 0 +} + +func (st *testStream) AddFailedTimes() { + return +} + +func (st *testStream) ResetFailedRimes() { + return +} + func makeDummyTestStreams(indexes []int) []sttypes.Stream { sts := make([]sttypes.Stream, 0, len(indexes)) diff --git a/p2p/stream/common/streammanager/interface_test.go b/p2p/stream/common/streammanager/interface_test.go index fc280b47ed..e308c3abab 100644 --- a/p2p/stream/common/streammanager/interface_test.go +++ b/p2p/stream/common/streammanager/interface_test.go @@ -70,6 +70,18 @@ func (st *testStream) ReadBytes() ([]byte, error) { return nil, nil } +func (st *testStream) FailedTimes() int { + return 0 +} + +func (st *testStream) AddFailedTimes() { + return +} + +func (st *testStream) ResetFailedRimes() { + return +} + func (st *testStream) Close() error { if st.closed { return errors.New("already closed") diff --git a/p2p/stream/common/streammanager/streammanager.go b/p2p/stream/common/streammanager/streammanager.go index 4b3d19b921..33270e1c7b 100644 --- a/p2p/stream/common/streammanager/streammanager.go +++ b/p2p/stream/common/streammanager/streammanager.go @@ -10,6 +10,7 @@ import ( "github.com/harmony-one/abool" "github.com/harmony-one/harmony/internal/utils" sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/harmony-one/harmony/shard" "github.com/libp2p/go-libp2p/core/network" libp2p_peer "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" @@ -73,6 +74,9 @@ func newStreamManager(pid sttypes.ProtoID, host host, pf peerFinder, handleStrea protoSpec, _ := sttypes.ProtoIDToProtoSpec(pid) + fmt.Println("my peer id: ", host.ID().String()) + fmt.Println("my proto id: ", pid) + return &streamManager{ myProtoID: pid, myProtoSpec: protoSpec, @@ -234,6 +238,9 @@ func (sm *streamManager) sanityCheckStream(st sttypes.Stream) error { if mySpec.ShardID != rmSpec.ShardID { return fmt.Errorf("unexpected shard ID: %v/%v", rmSpec.ShardID, mySpec.ShardID) } + if mySpec.ShardID == shard.BeaconChainShardID && !rmSpec.BeaconNode { + return fmt.Errorf("unexpected beacon node with shard ID: %v/%v", rmSpec.ShardID, mySpec.ShardID) + } return nil } @@ -323,7 +330,7 @@ func (sm *streamManager) discoverAndSetupStream(discCtx context.Context) (int, e } func (sm *streamManager) discover(ctx context.Context) (<-chan libp2p_peer.AddrInfo, error) { - protoID := string(sm.myProtoID) + protoID := sm.targetProtoID() discBatch := sm.config.DiscBatch if sm.config.HiCap-sm.streams.size() < sm.config.DiscBatch { discBatch = sm.config.HiCap - sm.streams.size() @@ -340,6 +347,14 @@ func (sm *streamManager) discover(ctx context.Context) (<-chan libp2p_peer.AddrI return sm.pf.FindPeers(ctx2, protoID, discBatch) } +func (sm *streamManager) targetProtoID() string { + targetSpec := sm.myProtoSpec + if targetSpec.ShardID == shard.BeaconChainShardID { // for beacon chain, only connect to beacon nodes + targetSpec.BeaconNode = true + } + return string(targetSpec.ToProtoID()) +} + func (sm *streamManager) setupStreamWithPeer(ctx context.Context, pid libp2p_peer.ID) error { timer := prometheus.NewTimer(setupStreamDuration.With(prometheus.Labels{"topic": string(sm.myProtoID)})) defer timer.ObserveDuration() @@ -347,7 +362,7 @@ func (sm *streamManager) setupStreamWithPeer(ctx context.Context, pid libp2p_pee nCtx, cancel := context.WithTimeout(ctx, connectTimeout) defer cancel() - st, err := sm.host.NewStream(nCtx, pid, protocol.ID(sm.myProtoID)) + st, err := sm.host.NewStream(nCtx, pid, protocol.ID(sm.targetProtoID())) if err != nil { return err } @@ -392,6 +407,10 @@ func (ss *streamSet) get(id sttypes.StreamID) (sttypes.Stream, bool) { ss.lock.RLock() defer ss.lock.RUnlock() + if id == "" { + return nil, false + } + st, ok := ss.streams[id] return st, ok } diff --git a/p2p/stream/protocols/sync/client.go b/p2p/stream/protocols/sync/client.go index 2523ef4f7e..7ad4798851 100644 --- a/p2p/stream/protocols/sync/client.go +++ b/p2p/stream/protocols/sync/client.go @@ -43,6 +43,36 @@ func (p *Protocol) GetBlocksByNumber(ctx context.Context, bns []uint64, opts ... return } +func (p *Protocol) GetRawBlocksByNumber(ctx context.Context, bns []uint64, opts ...Option) (blockBytes [][]byte, sigBytes [][]byte, stid sttypes.StreamID, err error) { + timer := p.doMetricClientRequest("getBlocksByNumber") + defer p.doMetricPostClientRequest("getBlocksByNumber", err, timer) + + if len(bns) == 0 { + err = fmt.Errorf("zero block numbers requested") + return + } + if len(bns) > GetBlocksByNumAmountCap { + err = fmt.Errorf("number of blocks exceed cap of %v", GetBlocksByNumAmountCap) + return + } + req := newGetBlocksByNumberRequest(bns) + resp, stid, err := p.rm.DoRequest(ctx, req, opts...) + if err != nil { + // At this point, error can be context canceled, context timed out, or waiting queue + // is already full. + return + } + + // Parse and return blocks + sResp, ok := resp.(*syncResponse) + if !ok || sResp == nil { + err = errors.New("not sync response") + return + } + blockBytes, sigBytes, err = req.parseBlockBytesAndSigs(sResp) + return +} + // GetCurrentBlockNumber get the current block number from remote node func (p *Protocol) GetCurrentBlockNumber(ctx context.Context, opts ...Option) (bn uint64, stid sttypes.StreamID, err error) { timer := p.doMetricClientRequest("getBlockNumber") diff --git a/p2p/stream/protocols/sync/const.go b/p2p/stream/protocols/sync/const.go index f536d4a78a..1e1fc612f0 100644 --- a/p2p/stream/protocols/sync/const.go +++ b/p2p/stream/protocols/sync/const.go @@ -17,6 +17,9 @@ const ( // See comments for GetBlocksByNumAmountCap. GetBlocksByHashesAmountCap = 10 + // MaxStreamFailures is the maximum allowed failures before stream gets removed + MaxStreamFailures = 3 + // minAdvertiseInterval is the minimum advertise interval minAdvertiseInterval = 1 * time.Minute diff --git a/p2p/stream/protocols/sync/protocol.go b/p2p/stream/protocols/sync/protocol.go index facdf601db..c63e6c45a5 100644 --- a/p2p/stream/protocols/sync/protocol.go +++ b/p2p/stream/protocols/sync/protocol.go @@ -2,11 +2,13 @@ package sync import ( "context" + "fmt" "strconv" "time" "github.com/ethereum/go-ethereum/event" "github.com/harmony-one/harmony/consensus/engine" + "github.com/harmony-one/harmony/core" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" "github.com/harmony-one/harmony/internal/utils" @@ -15,6 +17,7 @@ import ( "github.com/harmony-one/harmony/p2p/stream/common/requestmanager" "github.com/harmony-one/harmony/p2p/stream/common/streammanager" sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/harmony-one/harmony/shard" "github.com/hashicorp/go-version" libp2p_host "github.com/libp2p/go-libp2p/core/host" libp2p_network "github.com/libp2p/go-libp2p/core/network" @@ -39,12 +42,13 @@ var ( type ( // Protocol is the protocol for sync streaming Protocol struct { - chain engine.ChainReader // provide SYNC data - schedule shardingconfig.Schedule // provide schedule information - rl ratelimiter.RateLimiter // limit the incoming request rate - sm streammanager.StreamManager // stream management - rm requestmanager.RequestManager // deliver the response from stream - disc discovery.Discovery + chain engine.ChainReader // provide SYNC data + beaconNode bool // is beacon node or shard chain node + schedule shardingconfig.Schedule // provide schedule information + rl ratelimiter.RateLimiter // limit the incoming request rate + sm streammanager.StreamManager // stream management + rm requestmanager.RequestManager // deliver the response from stream + disc discovery.Discovery config Config logger zerolog.Logger @@ -74,13 +78,18 @@ type ( func NewProtocol(config Config) *Protocol { ctx, cancel := context.WithCancel(context.Background()) + isBeaconNode := config.Chain.ShardID() == shard.BeaconChainShardID + if _, ok := config.Chain.(*core.EpochChain); ok { + isBeaconNode = false + } sp := &Protocol{ - chain: config.Chain, - disc: config.Discovery, - config: config, - ctx: ctx, - cancel: cancel, - closeC: make(chan struct{}), + chain: config.Chain, + beaconNode: isBeaconNode, + disc: config.Discovery, + config: config, + ctx: ctx, + cancel: cancel, + closeC: make(chan struct{}), } smConfig := streammanager.Config{ SoftLoCap: config.SmSoftLowCap, @@ -162,6 +171,7 @@ func (p *Protocol) HandleStream(raw libp2p_network.Stream) { Msg("failed to add new stream") return } + fmt.Println("Node connected to", raw.Conn().RemotePeer().String(), "(", st.ProtoID(), ")") st.run() } @@ -219,18 +229,40 @@ func (p *Protocol) protoIDByVersion(v *version.Version) sttypes.ProtoID { NetworkType: p.config.Network, ShardID: p.config.ShardID, Version: v, + BeaconNode: p.beaconNode, } return spec.ToProtoID() } // RemoveStream removes the stream of the given stream ID +// TODO: add reason to parameters func (p *Protocol) RemoveStream(stID sttypes.StreamID) { - if stID == "" { - return - } st, exist := p.sm.GetStreamByID(stID) if exist && st != nil { + //TODO: log this incident with reason st.Close() + // stream manager removes this stream from the list and triggers discovery if number of streams are not enough + p.sm.RemoveStream(stID) //TODO: double check to see if this part is needed + } +} + +func (p *Protocol) StreamFailed(stID sttypes.StreamID, reason string) { + st, exist := p.sm.GetStreamByID(stID) + if exist && st != nil { + st.AddFailedTimes() + p.logger.Info(). + Str("stream ID", string(st.ID())). + Int("num failures", st.FailedTimes()). + Str("reason", reason). + Msg("stream failed") + if st.FailedTimes() >= MaxStreamFailures { + st.Close() + // stream manager removes this stream from the list and triggers discovery if number of streams are not enough + p.sm.RemoveStream(stID) //TODO: double check to see if this part is needed + p.logger.Warn(). + Str("stream ID", string(st.ID())). + Msg("stream removed") + } } } diff --git a/p2p/stream/types/stream.go b/p2p/stream/types/stream.go index 3abdf4f524..49860a2895 100644 --- a/p2p/stream/types/stream.go +++ b/p2p/stream/types/stream.go @@ -21,6 +21,9 @@ type Stream interface { ReadBytes() ([]byte, error) Close() error CloseOnExit() error + FailedTimes() int + AddFailedTimes() + ResetFailedRimes() } // BaseStream is the wrapper around @@ -34,14 +37,17 @@ type BaseStream struct { spec ProtoSpec specErr error specOnce sync.Once + + failedTimes int } // NewBaseStream creates BaseStream as the wrapper of libp2p Stream func NewBaseStream(st libp2p_network.Stream) *BaseStream { reader := bufio.NewReader(st) return &BaseStream{ - raw: st, - reader: reader, + raw: st, + reader: reader, + failedTimes: 0, } } @@ -72,6 +78,18 @@ func (st *BaseStream) Close() error { return st.raw.Reset() } +func (st *BaseStream) FailedTimes() int { + return st.failedTimes +} + +func (st *BaseStream) AddFailedTimes() { + st.failedTimes++ +} + +func (st *BaseStream) ResetFailedRimes() { + st.failedTimes = 0 +} + const ( maxMsgBytes = 20 * 1024 * 1024 // 20MB sizeBytes = 4 // uint32 diff --git a/p2p/stream/types/utils.go b/p2p/stream/types/utils.go index d096115fb7..bd8c6144af 100644 --- a/p2p/stream/types/utils.go +++ b/p2p/stream/types/utils.go @@ -11,7 +11,7 @@ import ( nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/hashicorp/go-version" - libp2p_proto "github.com/libp2p/go-libp2p/core/protocol" + libp2p_proto "github.com/libp2p/go-libp2p-core/protocol" "github.com/pkg/errors" ) @@ -20,10 +20,10 @@ const ( ProtoIDCommonPrefix = "harmony" // ProtoIDFormat is the format of stream protocol ID - ProtoIDFormat = "%s/%s/%s/%d/%s" + ProtoIDFormat = "%s/%s/%s/%d/%s/%d" // protoIDNumElem is the number of elements of the ProtoID. See comments in ProtoID - protoIDNumElem = 5 + protoIDNumElem = 6 ) // ProtoID is the protocol id for streaming, an alias of libp2p stream protocol ID。 @@ -32,6 +32,7 @@ const ( // 2. NetworkType - mainnet, testnet, stn, e.t.c. // 3. ShardID - shard ID of the current protocol. // 4. Version - Stream protocol version for backward compatibility. +// 5. BeaconNode - whether stream is from a beacon chain node or shard chain node type ProtoID libp2p_proto.ID // ProtoSpec is the un-serialized stream proto id specification @@ -43,12 +44,13 @@ type ProtoSpec struct { NetworkType nodeconfig.NetworkType ShardID nodeconfig.ShardID Version *version.Version + BeaconNode bool } // ToProtoID convert a ProtoSpec to ProtoID. func (spec ProtoSpec) ToProtoID() ProtoID { s := fmt.Sprintf(ProtoIDFormat, ProtoIDCommonPrefix, spec.Service, - spec.NetworkType, spec.ShardID, spec.Version.String()) + spec.NetworkType, spec.ShardID, spec.Version.String(), bool2int(spec.BeaconNode)) return ProtoID(s) } @@ -59,11 +61,12 @@ func ProtoIDToProtoSpec(id ProtoID) (ProtoSpec, error) { return ProtoSpec{}, errors.New("unexpected protocol size") } var ( - prefix = comps[0] - service = comps[1] - networkType = comps[2] - shardIDStr = comps[3] - versionStr = comps[4] + prefix = comps[0] + service = comps[1] + networkType = comps[2] + shardIDStr = comps[3] + versionStr = comps[4] + beaconnodeStr = comps[5] ) shardID, err := strconv.Atoi(shardIDStr) if err != nil { @@ -76,11 +79,16 @@ func ProtoIDToProtoSpec(id ProtoID) (ProtoSpec, error) { if err != nil { return ProtoSpec{}, errors.Wrap(err, "unexpected version string") } + isBeaconNode, err := strconv.Atoi(beaconnodeStr) + if err != nil { + return ProtoSpec{}, errors.Wrap(err, "invalid beacon node flag") + } return ProtoSpec{ Service: service, NetworkType: nodeconfig.NetworkType(networkType), ShardID: nodeconfig.ShardID(uint32(shardID)), Version: version, + BeaconNode: int2bool(isBeaconNode), }, nil } @@ -90,3 +98,14 @@ func GenReqID() uint64 { rand.Read(rnd[:]) return binary.BigEndian.Uint64(rnd[:]) } + +func bool2int(b bool) int { + if b { + return 1 + } + return 0 +} + +func int2bool(i int) bool { + return i > 0 +} From eb15d1384240c46b549954d2d2488f8a9d0935dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <“Gheis.Mohammadi@gmail.com”> Date: Tue, 27 Dec 2022 18:01:25 +0800 Subject: [PATCH 027/420] fix protocol tests --- api/service/stagedstreamsync/downloader.go | 3 ++- api/service/stagedstreamsync/staged_stream_sync.go | 6 +++--- p2p/stream/common/streammanager/interface_test.go | 2 +- p2p/stream/common/streammanager/streammanager.go | 2 +- .../common/streammanager/streammanager_test.go | 2 +- p2p/stream/protocols/sync/protocol_test.go | 12 +++++++----- p2p/stream/types/utils.go | 6 +++++- 7 files changed, 20 insertions(+), 13 deletions(-) diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go index 9fdd2f78f4..383447f982 100644 --- a/api/service/stagedstreamsync/downloader.go +++ b/api/service/stagedstreamsync/downloader.go @@ -32,7 +32,7 @@ type ( downloadC chan struct{} closeC chan struct{} ctx context.Context - cancel func() + cancel context.CancelFunc config Config logger zerolog.Logger @@ -70,6 +70,7 @@ func NewDownloader(host p2p.Host, bc core.BlockChain, config Config) *Downloader //TODO: use mem db should be in config file stagedSyncInstance, err := CreateStagedSync(ctx, bc, false, sp, config, logger, config.LogProgress) if err != nil { + cancel() return nil } diff --git a/api/service/stagedstreamsync/staged_stream_sync.go b/api/service/stagedstreamsync/staged_stream_sync.go index 1abd728498..11ad9e8aee 100644 --- a/api/service/stagedstreamsync/staged_stream_sync.go +++ b/api/service/stagedstreamsync/staged_stream_sync.go @@ -65,8 +65,8 @@ type StagedStreamSync struct { inserted int config Config logger zerolog.Logger - status status //TODO: merge this with currentSyncCycle - initSync bool // if sets to true, node start long range syncing + status *status //TODO: merge this with currentSyncCycle + initSync bool // if sets to true, node start long range syncing UseMemDB bool revertPoint *uint64 // used to run stages @@ -294,7 +294,7 @@ func New(ctx context.Context, db: db, protocol: protocol, gbm: nil, - status: status, + status: &status, inserted: 0, config: config, logger: logger, diff --git a/p2p/stream/common/streammanager/interface_test.go b/p2p/stream/common/streammanager/interface_test.go index e308c3abab..033273fd67 100644 --- a/p2p/stream/common/streammanager/interface_test.go +++ b/p2p/stream/common/streammanager/interface_test.go @@ -17,7 +17,7 @@ var _ StreamManager = &streamManager{} var ( myPeerID = makePeerID(0) - testProtoID = sttypes.ProtoID("harmony/sync/unitest/0/1.0.0") + testProtoID = sttypes.ProtoID("harmony/sync/unitest/0/1.0.0/1") ) const ( diff --git a/p2p/stream/common/streammanager/streammanager.go b/p2p/stream/common/streammanager/streammanager.go index 33270e1c7b..a341184981 100644 --- a/p2p/stream/common/streammanager/streammanager.go +++ b/p2p/stream/common/streammanager/streammanager.go @@ -410,7 +410,7 @@ func (ss *streamSet) get(id sttypes.StreamID) (sttypes.Stream, bool) { if id == "" { return nil, false } - + st, ok := ss.streams[id] return st, ok } diff --git a/p2p/stream/common/streammanager/streammanager_test.go b/p2p/stream/common/streammanager/streammanager_test.go index 5d82c8585f..2b49a5f16c 100644 --- a/p2p/stream/common/streammanager/streammanager_test.go +++ b/p2p/stream/common/streammanager/streammanager_test.go @@ -209,7 +209,7 @@ func TestStreamSet_numStreamsWithMinProtoID(t *testing.T) { pid1 = testProtoID numPid1 = 5 - pid2 = sttypes.ProtoID("harmony/sync/unitest/0/1.0.1") + pid2 = sttypes.ProtoID("harmony/sync/unitest/0/1.0.1/1") numPid2 = 10 ) diff --git a/p2p/stream/protocols/sync/protocol_test.go b/p2p/stream/protocols/sync/protocol_test.go index aff6691ec8..0e40f60175 100644 --- a/p2p/stream/protocols/sync/protocol_test.go +++ b/p2p/stream/protocols/sync/protocol_test.go @@ -15,16 +15,18 @@ func TestProtocol_Match(t *testing.T) { targetID string exp bool }{ - {"harmony/sync/unitest/0/1.0.1", true}, + {"harmony/sync/unitest/0/1.0.1/1", true}, + {"harmony/sync/unitest/0/1.0.1/0", true}, {"h123456", false}, - {"harmony/sync/unitest/0/0.9.9", false}, - {"harmony/epoch/unitest/0/1.0.1", false}, - {"harmony/sync/mainnet/0/1.0.1", false}, - {"harmony/sync/unitest/1/1.0.1", false}, + {"harmony/sync/unitest/0/0.9.9/1", false}, + {"harmony/epoch/unitest/0/1.0.1/1", false}, + {"harmony/sync/mainnet/0/1.0.1/1", false}, + {"harmony/sync/unitest/1/1.0.1/1", false}, } for i, test := range tests { p := &Protocol{ + beaconNode: true, config: Config{ Network: "unitest", ShardID: 0, diff --git a/p2p/stream/types/utils.go b/p2p/stream/types/utils.go index bd8c6144af..86eb882183 100644 --- a/p2p/stream/types/utils.go +++ b/p2p/stream/types/utils.go @@ -49,8 +49,12 @@ type ProtoSpec struct { // ToProtoID convert a ProtoSpec to ProtoID. func (spec ProtoSpec) ToProtoID() ProtoID { + var versionStr string + if spec.Version != nil { + versionStr = spec.Version.String() + } s := fmt.Sprintf(ProtoIDFormat, ProtoIDCommonPrefix, spec.Service, - spec.NetworkType, spec.ShardID, spec.Version.String(), bool2int(spec.BeaconNode)) + spec.NetworkType, spec.ShardID, versionStr, bool2int(spec.BeaconNode)) return ProtoID(s) } From 70a37800b53fbdef74b7aaea1af367790dce8226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <“Gheis.Mohammadi@gmail.com”> Date: Tue, 27 Dec 2022 18:26:22 +0800 Subject: [PATCH 028/420] fix spell --- p2p/stream/common/requestmanager/interface_test.go | 2 +- p2p/stream/common/streammanager/interface_test.go | 2 +- p2p/stream/types/stream.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/p2p/stream/common/requestmanager/interface_test.go b/p2p/stream/common/requestmanager/interface_test.go index 76399e859e..c51303ccba 100644 --- a/p2p/stream/common/requestmanager/interface_test.go +++ b/p2p/stream/common/requestmanager/interface_test.go @@ -122,7 +122,7 @@ func (st *testStream) AddFailedTimes() { return } -func (st *testStream) ResetFailedRimes() { +func (st *testStream) ResetFailedTimes() { return } diff --git a/p2p/stream/common/streammanager/interface_test.go b/p2p/stream/common/streammanager/interface_test.go index 033273fd67..5a9bb44366 100644 --- a/p2p/stream/common/streammanager/interface_test.go +++ b/p2p/stream/common/streammanager/interface_test.go @@ -78,7 +78,7 @@ func (st *testStream) AddFailedTimes() { return } -func (st *testStream) ResetFailedRimes() { +func (st *testStream) ResetFailedTimes() { return } diff --git a/p2p/stream/types/stream.go b/p2p/stream/types/stream.go index 49860a2895..18b47f6158 100644 --- a/p2p/stream/types/stream.go +++ b/p2p/stream/types/stream.go @@ -23,7 +23,7 @@ type Stream interface { CloseOnExit() error FailedTimes() int AddFailedTimes() - ResetFailedRimes() + ResetFailedTimes() } // BaseStream is the wrapper around @@ -86,7 +86,7 @@ func (st *BaseStream) AddFailedTimes() { st.failedTimes++ } -func (st *BaseStream) ResetFailedRimes() { +func (st *BaseStream) ResetFailedTimes() { st.failedTimes = 0 } From adf8ad769e04f5dd05e4b20f24fd04bb8e8f11ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <“Gheis.Mohammadi@gmail.com”> Date: Tue, 27 Dec 2022 19:50:09 +0800 Subject: [PATCH 029/420] remove unused struct --- api/service/stagedstreamsync/staged_stream_sync.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/api/service/stagedstreamsync/staged_stream_sync.go b/api/service/stagedstreamsync/staged_stream_sync.go index 11ad9e8aee..a5fa011f43 100644 --- a/api/service/stagedstreamsync/staged_stream_sync.go +++ b/api/service/stagedstreamsync/staged_stream_sync.go @@ -9,7 +9,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/event" "github.com/harmony-one/harmony/core" - "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/utils" syncproto "github.com/harmony-one/harmony/p2p/stream/protocols/sync" sttypes "github.com/harmony-one/harmony/p2p/stream/types" @@ -87,13 +86,6 @@ type StagedStreamSync struct { evtDownloadStartedSubscribed bool } -// BlockWithSig the serialization structure for request DownloaderRequest_BLOCKWITHSIG -// The block is encoded as block + commit signature -type BlockWithSig struct { - Block *types.Block - CommitSigAndBitmap []byte -} - type Timing struct { isRevert bool isCleanUp bool @@ -185,7 +177,7 @@ func (s *StagedStreamSync) IsAfter(stage1, stage2 SyncStageID) bool { func (s *StagedStreamSync) RevertTo(revertPoint uint64, invalidBlockNumber uint64, invalidBlockHash common.Hash, invalidBlockStreamID sttypes.StreamID) { utils.Logger().Info(). - Interface("invalidBlockNumber", invalidBlockNumber). + Uint64("invalidBlockNumber", invalidBlockNumber). Interface("invalidBlockHash", invalidBlockHash). Interface("invalidBlockStreamID", invalidBlockStreamID). Uint64("revertPoint", revertPoint). From 28010ac083b7491a1182ea6f87d2a456a25a0469 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <“Gheis.Mohammadi@gmail.com”> Date: Wed, 28 Dec 2022 13:34:35 +0800 Subject: [PATCH 030/420] fix rosetta test --- rosetta/infra/harmony-mainnet.conf | 3 ++- rosetta/infra/harmony-pstn.conf | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/rosetta/infra/harmony-mainnet.conf b/rosetta/infra/harmony-mainnet.conf index 534b80887f..ddb2f87eea 100644 --- a/rosetta/infra/harmony-mainnet.conf +++ b/rosetta/infra/harmony-mainnet.conf @@ -1,4 +1,4 @@ -Version = "2.5.9" +Version = "2.5.10" [BLSKeys] KMSConfigFile = "" @@ -68,6 +68,7 @@ Version = "2.5.9" MaxConnsPerIP = 10 MaxPeers = 0 Port = 9000 + WaitForEachPeerToConnect = false [Pprof] Enabled = false diff --git a/rosetta/infra/harmony-pstn.conf b/rosetta/infra/harmony-pstn.conf index ed4c116c6b..1c029f8281 100644 --- a/rosetta/infra/harmony-pstn.conf +++ b/rosetta/infra/harmony-pstn.conf @@ -1,4 +1,4 @@ -Version = "2.5.9" +Version = "2.5.10" [BLSKeys] KMSConfigFile = "" @@ -68,6 +68,7 @@ Version = "2.5.9" MaxConnsPerIP = 10 MaxPeers = 0 Port = 9000 + WaitForEachPeerToConnect = false [Pprof] Enabled = false From cc0f3d459d77bca19755a1ce10d6d0727acc242b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <“Gheis.Mohammadi@gmail.com”> Date: Wed, 28 Dec 2022 18:41:09 +0800 Subject: [PATCH 031/420] add comments and refactor verify sig --- api/service/stagedstreamsync/const.go | 4 +- api/service/stagedstreamsync/downloader.go | 56 +---------------- api/service/stagedstreamsync/helpers.go | 14 +++++ api/service/stagedstreamsync/sig_verify.go | 60 +++++++++++++++++++ .../stagedstreamsync/staged_stream_sync.go | 32 +++++----- 5 files changed, 95 insertions(+), 71 deletions(-) create mode 100644 api/service/stagedstreamsync/sig_verify.go diff --git a/api/service/stagedstreamsync/const.go b/api/service/stagedstreamsync/const.go index 721b44d9f5..ed68d80b13 100644 --- a/api/service/stagedstreamsync/const.go +++ b/api/service/stagedstreamsync/const.go @@ -21,11 +21,11 @@ const ( SoftQueueCap int = 100 // DefaultConcurrency is the default settings for concurrency - DefaultConcurrency = 4 + DefaultConcurrency int = 4 // ShortRangeTimeout is the timeout for each short range sync, which allow short range sync // to restart automatically when stuck in `getBlockHashes` - ShortRangeTimeout = 1 * time.Minute + ShortRangeTimeout time.Duration = 1 * time.Minute ) type ( diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go index 383447f982..9883c8e8f7 100644 --- a/api/service/stagedstreamsync/downloader.go +++ b/api/service/stagedstreamsync/downloader.go @@ -6,13 +6,9 @@ import ( "time" "github.com/ethereum/go-ethereum/event" - "github.com/pkg/errors" "github.com/rs/zerolog" "github.com/harmony-one/harmony/core" - "github.com/harmony-one/harmony/core/types" - "github.com/harmony-one/harmony/crypto/bls" - "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" @@ -128,7 +124,7 @@ func (d *Downloader) NumPeers() int { return d.syncProtocol.NumStreams() } -// IsSyncing returns the current sync status +// SyncStatus returns the current sync status func (d *Downloader) SyncStatus() (bool, uint64, uint64) { syncing, target := d.stagedSyncInstance.status.get() if !syncing { @@ -252,53 +248,3 @@ func (d *Downloader) loop() { } } } - -var emptySigVerifyErr *sigVerifyErr - -type sigVerifyErr struct { - err error -} - -func (e *sigVerifyErr) Error() string { - return fmt.Sprintf("[VerifyHeaderSignature] %v", e.err.Error()) -} - -func verifyAndInsertBlocks(bc blockChain, blocks types.Blocks) (int, error) { - for i, block := range blocks { - if err := verifyAndInsertBlock(bc, block, blocks[i+1:]...); err != nil { - return i, err - } - } - return len(blocks), nil -} - -func verifyAndInsertBlock(bc blockChain, block *types.Block, nextBlocks ...*types.Block) error { - var ( - sigBytes bls.SerializedSignature - bitmap []byte - err error - ) - if len(nextBlocks) > 0 { - // get commit sig from the next block - next := nextBlocks[0] - sigBytes = next.Header().LastCommitSignature() - bitmap = next.Header().LastCommitBitmap() - } else { - // get commit sig from current block - sigBytes, bitmap, err = chain.ParseCommitSigAndBitmap(block.GetCurrentCommitSig()) - if err != nil { - return errors.Wrap(err, "parse commitSigAndBitmap") - } - } - - if err := bc.Engine().VerifyHeaderSignature(bc, block.Header(), sigBytes, bitmap); err != nil { - return &sigVerifyErr{err} - } - if err := bc.Engine().VerifyHeader(bc, block.Header(), true); err != nil { - return errors.Wrap(err, "[VerifyHeader]") - } - if _, err := bc.InsertChain(types.Blocks{block}, false); err != nil { - return errors.Wrap(err, "[InsertChain]") - } - return nil -} diff --git a/api/service/stagedstreamsync/helpers.go b/api/service/stagedstreamsync/helpers.go index cd6fd8f6f0..9e986a538b 100644 --- a/api/service/stagedstreamsync/helpers.go +++ b/api/service/stagedstreamsync/helpers.go @@ -112,3 +112,17 @@ func countHashMaxVote(m map[sttypes.StreamID]common.Hash, whitelist map[sttypes. } return res, nextWl } + +func ByteCount(b uint64) string { + const unit = 1024 + if b < unit { + return fmt.Sprintf("%dB", b) + } + div, exp := uint64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f%cB", + float64(b)/float64(div), "KMGTPE"[exp]) +} diff --git a/api/service/stagedstreamsync/sig_verify.go b/api/service/stagedstreamsync/sig_verify.go new file mode 100644 index 0000000000..649c6eaec1 --- /dev/null +++ b/api/service/stagedstreamsync/sig_verify.go @@ -0,0 +1,60 @@ +package stagedstreamsync + +import ( + "fmt" + + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/crypto/bls" + "github.com/harmony-one/harmony/internal/chain" + "github.com/pkg/errors" +) + +var emptySigVerifyErr *sigVerifyErr + +type sigVerifyErr struct { + err error +} + +func (e *sigVerifyErr) Error() string { + return fmt.Sprintf("[VerifyHeaderSignature] %v", e.err.Error()) +} + +func verifyAndInsertBlocks(bc blockChain, blocks types.Blocks) (int, error) { + for i, block := range blocks { + if err := verifyAndInsertBlock(bc, block, blocks[i+1:]...); err != nil { + return i, err + } + } + return len(blocks), nil +} + +func verifyAndInsertBlock(bc blockChain, block *types.Block, nextBlocks ...*types.Block) error { + var ( + sigBytes bls.SerializedSignature + bitmap []byte + err error + ) + if len(nextBlocks) > 0 { + // get commit sig from the next block + next := nextBlocks[0] + sigBytes = next.Header().LastCommitSignature() + bitmap = next.Header().LastCommitBitmap() + } else { + // get commit sig from current block + sigBytes, bitmap, err = chain.ParseCommitSigAndBitmap(block.GetCurrentCommitSig()) + if err != nil { + return errors.Wrap(err, "parse commitSigAndBitmap") + } + } + + if err := bc.Engine().VerifyHeaderSignature(bc, block.Header(), sigBytes, bitmap); err != nil { + return &sigVerifyErr{err} + } + if err := bc.Engine().VerifyHeader(bc, block.Header(), true); err != nil { + return errors.Wrap(err, "[VerifyHeader]") + } + if _, err := bc.InsertChain(types.Blocks{block}, false); err != nil { + return errors.Wrap(err, "[InsertChain]") + } + return nil +} diff --git a/api/service/stagedstreamsync/staged_stream_sync.go b/api/service/stagedstreamsync/staged_stream_sync.go index a5fa011f43..9603f5b06b 100644 --- a/api/service/stagedstreamsync/staged_stream_sync.go +++ b/api/service/stagedstreamsync/staged_stream_sync.go @@ -175,6 +175,7 @@ func (s *StagedStreamSync) IsAfter(stage1, stage2 SyncStageID) bool { return idx1 > idx2 } +// RevertTo sets the revert point func (s *StagedStreamSync) RevertTo(revertPoint uint64, invalidBlockNumber uint64, invalidBlockHash common.Hash, invalidBlockStreamID sttypes.StreamID) { utils.Logger().Info(). Uint64("invalidBlockNumber", invalidBlockNumber). @@ -195,10 +196,12 @@ func (s *StagedStreamSync) Done() { s.revertPoint = nil } +// IsDone returns true if last stage have been done func (s *StagedStreamSync) IsDone() bool { return s.currentStage >= uint(len(s.stages)) && s.revertPoint == nil } +// SetCurrentStage sets the current stage to a given stage id func (s *StagedStreamSync) SetCurrentStage(id SyncStageID) error { for i, stage := range s.stages { if stage.ID == id { @@ -210,6 +213,7 @@ func (s *StagedStreamSync) SetCurrentStage(id SyncStageID) error { return ErrStageNotFound } +// StageState retrieves the latest stage state from db func (s *StagedStreamSync) StageState(stage SyncStageID, tx kv.Tx, db kv.RwDB) (*StageState, error) { var blockNum uint64 var err error @@ -226,6 +230,7 @@ func (s *StagedStreamSync) StageState(stage SyncStageID, tx kv.Tx, db kv.RwDB) ( return &StageState{s, stage, blockNum}, nil } +// cleanUp cleans up the stage by calling pruneStage func (s *StagedStreamSync) cleanUp(fromStage int, db kv.RwDB, tx kv.RwTx, firstCycle bool) error { found := false for i := 0; i < len(s.pruningOrder); i++ { @@ -242,6 +247,7 @@ func (s *StagedStreamSync) cleanUp(fromStage int, db kv.RwDB, tx kv.RwTx, firstC return nil } +// New creates a new StagedStreamSync instance func New(ctx context.Context, bc core.BlockChain, db kv.RwDB, @@ -299,6 +305,7 @@ func New(ctx context.Context, } } +// doGetCurrentNumberRequest returns estimated current block number and corresponding stream func (s *StagedStreamSync) doGetCurrentNumberRequest() (uint64, sttypes.StreamID, error) { ctx, cancel := context.WithTimeout(s.ctx, 10*time.Second) defer cancel() @@ -310,11 +317,13 @@ func (s *StagedStreamSync) doGetCurrentNumberRequest() (uint64, sttypes.StreamID return bn, stid, nil } +// promLabels returns a prometheus labels for current shard id func (s *StagedStreamSync) promLabels() prometheus.Labels { sid := s.bc.ShardID() return prometheus.Labels{"ShardID": fmt.Sprintf("%d", sid)} } +// checkHaveEnoughStreams checks whether node is connected to certain number of streams func (s *StagedStreamSync) checkHaveEnoughStreams() error { numStreams := s.protocol.NumStreams() if numStreams < s.config.MinStreams { @@ -324,6 +333,7 @@ func (s *StagedStreamSync) checkHaveEnoughStreams() error { return nil } +// SetNewContext sets a new context for all stages func (s *StagedStreamSync) SetNewContext(ctx context.Context) error { for _, s := range s.stages { s.Handler.SetStageContext(ctx) @@ -331,6 +341,7 @@ func (s *StagedStreamSync) SetNewContext(ctx context.Context) error { return nil } +// Run runs a full cycle of stages func (s *StagedStreamSync) Run(db kv.RwDB, tx kv.RwTx, firstCycle bool) error { s.prevRevertPoint = nil s.timings = s.timings[:0] @@ -395,6 +406,7 @@ func (s *StagedStreamSync) Run(db kv.RwDB, tx kv.RwTx, firstCycle bool) error { return nil } +// CreateView creates a view for a given db func CreateView(ctx context.Context, db kv.RwDB, tx kv.Tx, f func(tx kv.Tx) error) error { if tx != nil { return f(tx) @@ -404,20 +416,7 @@ func CreateView(ctx context.Context, db kv.RwDB, tx kv.Tx, f func(tx kv.Tx) erro }) } -func ByteCount(b uint64) string { - const unit = 1024 - if b < unit { - return fmt.Sprintf("%dB", b) - } - div, exp := uint64(unit), 0 - for n := b / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f%cB", - float64(b)/float64(div), "KMGTPE"[exp]) -} - +// printLogs prints all timing logs func printLogs(tx kv.RwTx, timings []Timing) error { var logCtx []interface{} count := 0 @@ -463,6 +462,7 @@ func printLogs(tx kv.RwTx, timings []Timing) error { return nil } +// runStage executes stage func (s *StagedStreamSync) runStage(stage *Stage, db kv.RwDB, tx kv.RwTx, firstCycle bool, invalidBlockRevert bool) (err error) { start := time.Now() stageState, err := s.StageState(stage.ID, tx, db) @@ -489,6 +489,7 @@ func (s *StagedStreamSync) runStage(stage *Stage, db kv.RwDB, tx kv.RwTx, firstC return nil } +// revertStage reverts stage func (s *StagedStreamSync) revertStage(firstCycle bool, stage *Stage, db kv.RwDB, tx kv.RwTx) error { start := time.Now() stageState, err := s.StageState(stage.ID, tx, db) @@ -521,6 +522,7 @@ func (s *StagedStreamSync) revertStage(firstCycle bool, stage *Stage, db kv.RwDB return nil } +// pruneStage cleans up the stage and logs the timing func (s *StagedStreamSync) pruneStage(firstCycle bool, stage *Stage, db kv.RwDB, tx kv.RwTx) error { start := time.Now() @@ -566,6 +568,7 @@ func (s *StagedStreamSync) DisableAllStages() []SyncStageID { return backupEnabledIds } +// DisableStages disables stages by a set of given stage IDs func (s *StagedStreamSync) DisableStages(ids ...SyncStageID) { for i := range s.stages { for _, id := range ids { @@ -577,6 +580,7 @@ func (s *StagedStreamSync) DisableStages(ids ...SyncStageID) { } } +// EnableStages enables stages by a set of given stage IDs func (s *StagedStreamSync) EnableStages(ids ...SyncStageID) { for i := range s.stages { for _, id := range ids { From 088ae63713e46256e174b0baeee809c36647b199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Wed, 28 Dec 2022 19:02:37 +0800 Subject: [PATCH 032/420] add comments, remove extra function --- api/service/stagedstreamsync/stages.go | 11 +++-------- api/service/stagedstreamsync/syncing.go | 1 + 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/api/service/stagedstreamsync/stages.go b/api/service/stagedstreamsync/stages.go index dc607416b8..55681d68f5 100644 --- a/api/service/stagedstreamsync/stages.go +++ b/api/service/stagedstreamsync/stages.go @@ -16,6 +16,7 @@ const ( Finish SyncStageID = "Finish" // Nominal stage after all other stages ) +// GetStageName returns the stage name in string func GetStageName(stage string, isBeacon bool, prune bool) string { name := stage if isBeacon { @@ -27,18 +28,11 @@ func GetStageName(stage string, isBeacon bool, prune bool) string { return name } +// GetStageID returns the stage name in bytes func GetStageID(stage SyncStageID, isBeacon bool, prune bool) []byte { return []byte(GetStageName(string(stage), isBeacon, prune)) } -func GetBucketName(bucketName string, isBeacon bool) string { - name := bucketName - if isBeacon { - name = "Beacon" + name - } - return name -} - // GetStageProgress retrieves saved progress of a given sync stage from the database func GetStageProgress(db kv.Getter, stage SyncStageID, isBeacon bool) (uint64, error) { stgID := GetStageID(stage, isBeacon, false) @@ -65,6 +59,7 @@ func GetStageCleanUpProgress(db kv.Getter, stage SyncStageID, isBeacon bool) (ui return unmarshalData(v) } +// SaveStageCleanUpProgress stores the progress of the clean up for a given sync stage to the database func SaveStageCleanUpProgress(db kv.Putter, stage SyncStageID, isBeacon bool, progress uint64) error { stgID := GetStageID(stage, isBeacon, true) return db.Put(kv.SyncStageProgress, stgID, marshalData(progress)) diff --git a/api/service/stagedstreamsync/syncing.go b/api/service/stagedstreamsync/syncing.go index 676eb3551e..046ba6843b 100644 --- a/api/service/stagedstreamsync/syncing.go +++ b/api/service/stagedstreamsync/syncing.go @@ -134,6 +134,7 @@ func initDB(ctx context.Context, mainDB kv.RwDB, dbs []kv.RwDB, concurrency int) return nil } +// GetBlockDbPath returns the path of the cache database which stores blocks func GetBlockDbPath(beacon bool, loopID int) string { if beacon { if loopID >= 0 { From 69c896bdd3e7c4f56c1f554e855e988c10689296 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Wed, 28 Dec 2022 19:19:27 +0800 Subject: [PATCH 033/420] add comment --- api/service/stagedstreamsync/stage.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/service/stagedstreamsync/stage.go b/api/service/stagedstreamsync/stage.go index 8f2040a789..832cf4e3c7 100644 --- a/api/service/stagedstreamsync/stage.go +++ b/api/service/stagedstreamsync/stage.go @@ -96,6 +96,7 @@ func (u *RevertState) Done(db kv.Putter) error { return SaveStageProgress(db, u.ID, u.state.isBeacon, u.RevertPoint) } +// CleanUpState contains states of cleanup process for a specific stage type CleanUpState struct { ID SyncStageID ForwardProgress uint64 // progress of stage forward move From 29d28f20a8cff1678a7207d60bdca979c8483899 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Thu, 29 Dec 2022 11:36:58 +0800 Subject: [PATCH 034/420] refactor errors, rename metrics --- api/service/stagedstreamsync/beacon_helper.go | 9 ++-- .../stagedstreamsync/block_by_hash_manager.go | 2 +- api/service/stagedstreamsync/downloader.go | 4 +- api/service/stagedstreamsync/errors.go | 53 ++++++------------- api/service/stagedstreamsync/helpers.go | 5 +- api/service/stagedstreamsync/metric.go | 14 ++--- .../stagedstreamsync/short_range_helper.go | 11 ++-- api/service/stagedstreamsync/stage.go | 4 -- api/service/stagedstreamsync/stage_bodies.go | 2 +- api/service/stagedstreamsync/syncing.go | 2 +- api/service/stagedstreamsync/types.go | 2 +- hmy/downloader/metric.go | 14 ++--- 12 files changed, 51 insertions(+), 71 deletions(-) diff --git a/api/service/stagedstreamsync/beacon_helper.go b/api/service/stagedstreamsync/beacon_helper.go index 4a77020160..2f684e0e27 100644 --- a/api/service/stagedstreamsync/beacon_helper.go +++ b/api/service/stagedstreamsync/beacon_helper.go @@ -42,7 +42,7 @@ func newBeaconHelper(bc blockChain, blockC <-chan *types.Block, insertHook func( insertC: make(chan insertTask, 1), closeC: make(chan struct{}), logger: utils.Logger().With(). - Str("module", "downloader"). + Str("module", "staged stream sync"). Str("sub-module", "beacon helper"). Logger(), } @@ -78,7 +78,8 @@ func (bh *beaconHelper) loop() { inserted, bn, err := bh.insertLastMileBlocks() numBlocksInsertedBeaconHelperCounter.Add(float64(inserted)) if err != nil { - bh.logger.Error().Err(err).Msg(WrapStagedSyncMsg("insert last mile blocks error")) + bh.logger.Error().Err(err). + Msg(WrapStagedSyncMsg("insert last mile blocks error")) continue } bh.logger.Info().Int("inserted", inserted). @@ -127,7 +128,9 @@ func (bh *beaconHelper) insertLastMileBlocks() (inserted int, bn uint64, err err bn-- return } - bh.logger.Info().Uint64("number", b.NumberU64()).Msg(WrapStagedSyncMsg("Inserted block from beacon pub-sub")) + bh.logger.Info(). + Uint64("number", b.NumberU64()). + Msg(WrapStagedSyncMsg("Inserted block from beacon pub-sub")) if bh.insertHook != nil { bh.insertHook() diff --git a/api/service/stagedstreamsync/block_by_hash_manager.go b/api/service/stagedstreamsync/block_by_hash_manager.go index 86abc22ed7..4179bbaed1 100644 --- a/api/service/stagedstreamsync/block_by_hash_manager.go +++ b/api/service/stagedstreamsync/block_by_hash_manager.go @@ -34,7 +34,7 @@ func (m *getBlocksByHashManager) getNextHashes() ([]common.Hash, []sttypes.Strea num := m.numBlocksPerRequest() hashes := make([]common.Hash, 0, num) if len(m.whitelist) == 0 { - return nil, nil, errors.New("empty white list") + return nil, nil, ErrEmptyWhitelist } for _, hash := range m.hashes { diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go index 9883c8e8f7..bb3b8c82eb 100644 --- a/api/service/stagedstreamsync/downloader.go +++ b/api/service/stagedstreamsync/downloader.go @@ -59,7 +59,9 @@ func NewDownloader(host p2p.Host, bc core.BlockChain, config Config) *Downloader bh = newBeaconHelper(bc, config.BHConfig.BlockC, config.BHConfig.InsertHook) } - logger := utils.Logger().With().Str("module", "StagedStreamSync").Uint32("ShardID", bc.ShardID()).Logger() + logger := utils.Logger().With(). + Str("module", "staged stream sync"). + Uint32("ShardID", bc.ShardID()).Logger() ctx, cancel := context.WithCancel(context.Background()) diff --git a/api/service/stagedstreamsync/errors.go b/api/service/stagedstreamsync/errors.go index e4828003d2..d18020dd06 100644 --- a/api/service/stagedstreamsync/errors.go +++ b/api/service/stagedstreamsync/errors.go @@ -6,43 +6,22 @@ import ( // Errors ... var ( - ErrRegistrationFail = WrapStagedSyncError("registration failed") - ErrGetBlock = WrapStagedSyncError("get block failed") - ErrGetBlockHash = WrapStagedSyncError("get block hash failed") - ErrGetConsensusHashes = WrapStagedSyncError("get consensus hashes failed") - ErrGenStateSyncTaskQueue = WrapStagedSyncError("generate state sync task queue failed") - ErrDownloadBlocks = WrapStagedSyncError("get download blocks failed") - ErrUpdateBlockAndStatus = WrapStagedSyncError("update block and status failed") - ErrGenerateNewState = WrapStagedSyncError("get generate new state failed") - ErrFetchBlockHashProgressFail = WrapStagedSyncError("fetch cache progress for block hashes stage failed") - ErrFetchCachedBlockHashFail = WrapStagedSyncError("fetch cached block hashes failed") - ErrNotEnoughBlockHashes = WrapStagedSyncError("peers haven't sent all requested block hashes") - ErrRetrieveCachedProgressFail = WrapStagedSyncError("retrieving cache progress for block hashes stage failed") - ErrRetrieveCachedHashProgressFail = WrapStagedSyncError("retrieving cache progress for block hashes stage failed") - ErrSaveBlockHashesProgressFail = WrapStagedSyncError("saving progress for block hashes stage failed") - ErrSaveCachedBlockHashesProgressFail = WrapStagedSyncError("saving cache progress for block hashes stage failed") - ErrSavingCacheLastBlockHashFail = WrapStagedSyncError("saving cache last block hash for block hashes stage failed") - ErrCachingBlockHashFail = WrapStagedSyncError("caching downloaded block hashes failed") - ErrCommitTransactionFail = WrapStagedSyncError("failed to write db commit") - ErrUnexpectedNumberOfBlocks = WrapStagedSyncError("unexpected number of block delivered") - ErrSavingBodiesProgressFail = WrapStagedSyncError("saving progress for block bodies stage failed") - ErrAddTasksToQueueFail = WrapStagedSyncError("cannot add task to queue") - ErrSavingCachedBodiesProgressFail = WrapStagedSyncError("saving cache progress for blocks stage failed") - ErrRetrievingCachedBodiesProgressFail = WrapStagedSyncError("retrieving cache progress for blocks stage failed") - ErrNoConnectedPeers = WrapStagedSyncError("haven't connected to any peer yet!") - ErrNotEnoughConnectedPeers = WrapStagedSyncError("not enough connected peers") - ErrSaveStateProgressFail = WrapStagedSyncError("saving progress for block States stage failed") - ErrPruningCursorCreationFail = WrapStagedSyncError("failed to create cursor for pruning") - ErrInvalidBlockNumber = WrapStagedSyncError("invalid block number") - ErrInvalidBlockBytes = WrapStagedSyncError("invalid block bytes to insert into chain") - ErrAddTaskFailed = WrapStagedSyncError("cannot add task to queue") - ErrNodeNotEnoughBlockHashes = WrapStagedSyncError("some of the nodes didn't provide all block hashes") - ErrCachingBlocksFail = WrapStagedSyncError("caching downloaded block bodies failed") - ErrSaveBlocksFail = WrapStagedSyncError("save downloaded block bodies failed") - ErrStageNotFound = WrapStagedSyncError("stage not found") - ErrSomeNodesNotReady = WrapStagedSyncError("some nodes are not ready") - ErrSomeNodesBlockHashFail = WrapStagedSyncError("some nodes failed to download block hashes") - ErrMaxPeerHeightFail = WrapStagedSyncError("get max peer height failed") + ErrSavingBodiesProgressFail = WrapStagedSyncError("saving progress for block bodies stage failed") + ErrSaveStateProgressFail = WrapStagedSyncError("saving progress for block States stage failed") + ErrInvalidBlockNumber = WrapStagedSyncError("invalid block number") + ErrInvalidBlockBytes = WrapStagedSyncError("invalid block bytes to insert into chain") + ErrStageNotFound = WrapStagedSyncError("stage not found") + ErrUnexpectedNumberOfBlockHashes = WrapStagedSyncError("unexpected number of getBlocksByHashes result") + ErrUnexpectedBlockHashes = WrapStagedSyncError("unexpected get block hashes result delivered") + ErrNilBlock = WrapStagedSyncError("nil block found") + ErrNotEnoughStreams = WrapStagedSyncError("not enough streams") + ErrParseCommitSigAndBitmapFail = WrapStagedSyncError("parse commitSigAndBitmap failed") + ErrVerifyHeaderFail = WrapStagedSyncError("verify header failed") + ErrInsertChainFail = WrapStagedSyncError("insert to chain failed") + ErrZeroBlockResponse = WrapStagedSyncError("zero block number response from remote nodes") + ErrEmptyWhitelist = WrapStagedSyncError("empty white list") + ErrWrongGetBlockNumberType = WrapStagedSyncError("wrong type of getBlockNumber interface") + ErrSaveBlocksToDbFailed = WrapStagedSyncError("saving downloaded blocks to db failed") ) // WrapStagedSyncError wraps errors for staged sync and returns error object diff --git a/api/service/stagedstreamsync/helpers.go b/api/service/stagedstreamsync/helpers.go index 9e986a538b..75e504214f 100644 --- a/api/service/stagedstreamsync/helpers.go +++ b/api/service/stagedstreamsync/helpers.go @@ -8,7 +8,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/harmony/core/types" sttypes "github.com/harmony-one/harmony/p2p/stream/types" - "github.com/pkg/errors" ) func marshalData(blockNumber uint64) []byte { @@ -61,11 +60,11 @@ func computeBlockNumberByMaxVote(votes map[sttypes.StreamID]uint64) uint64 { func checkGetBlockByHashesResult(blocks []*types.Block, hashes []common.Hash) error { if len(blocks) != len(hashes) { - return errors.New("unexpected number of getBlocksByHashes result") + return ErrUnexpectedNumberOfBlockHashes } for i, block := range blocks { if block == nil { - return errors.New("nil block found") + return ErrNilBlock } if block.Hash() != hashes[i] { return fmt.Errorf("unexpected block hash: %x / %x", block.Hash(), hashes[i]) diff --git a/api/service/stagedstreamsync/metric.go b/api/service/stagedstreamsync/metric.go index 1bc11a227f..9437cc09e3 100644 --- a/api/service/stagedstreamsync/metric.go +++ b/api/service/stagedstreamsync/metric.go @@ -23,7 +23,7 @@ var ( consensusTriggeredDownloadCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "staged_stream_sync", Name: "consensus_trigger", Help: "number of times consensus triggered download task", }, @@ -33,7 +33,7 @@ var ( longRangeSyncedBlockCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "staged_stream_sync", Name: "num_blocks_synced_long_range", Help: "number of blocks synced in long range sync", }, @@ -43,7 +43,7 @@ var ( longRangeFailInsertedBlockCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "staged_stream_sync", Name: "num_blocks_failed_long_range", Help: "number of blocks failed to insert into change in long range sync", }, @@ -53,7 +53,7 @@ var ( numShortRangeCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "staged_stream_sync", Name: "num_short_range", Help: "number of short range sync is triggered", }, @@ -63,7 +63,7 @@ var ( numFailedDownloadCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "staged_stream_sync", Name: "failed_download", Help: "number of downloading is failed", }, @@ -73,7 +73,7 @@ var ( numBlocksInsertedShortRangeHistogramVec = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "staged_stream_sync", Name: "num_blocks_inserted_short_range", Help: "number of blocks inserted for each short range sync", // Buckets: 0, 1, 2, 4, +INF (capped at 10) @@ -85,7 +85,7 @@ var ( numBlocksInsertedBeaconHelperCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "staged_stream_sync", Name: "num_blocks_inserted_beacon_helper", Help: "number of blocks inserted from beacon helper", }, diff --git a/api/service/stagedstreamsync/short_range_helper.go b/api/service/stagedstreamsync/short_range_helper.go index f8dd2d28ea..210d1aa0c8 100644 --- a/api/service/stagedstreamsync/short_range_helper.go +++ b/api/service/stagedstreamsync/short_range_helper.go @@ -134,7 +134,7 @@ func (sh *srHelper) getBlocksByHashes(hashes []common.Hash, whitelist []sttypes. func (sh *srHelper) checkPrerequisites() error { if sh.syncProtocol.NumStreams() < sh.config.Concurrency { - return errors.New("not enough streams") + return ErrNotEnoughStreams } return nil } @@ -166,12 +166,11 @@ func (sh *srHelper) doGetBlockHashesRequest(bns []uint64) ([]common.Hash, sttype return nil, stid, err } if len(hashes) != len(bns) { - err := errors.New("unexpected get block hashes result delivered") - sh.logger.Warn().Err(err). + sh.logger.Warn().Err(ErrUnexpectedBlockHashes). Str("stream", string(stid)). Msg(WrapStagedSyncMsg("failed to doGetBlockHashesRequest")) sh.syncProtocol.StreamFailed(stid, "unexpected get block hashes result delivered") - return nil, stid, err + return nil, stid, ErrUnexpectedBlockHashes } return hashes, stid, nil } @@ -182,7 +181,9 @@ func (sh *srHelper) doGetBlocksByNumbersRequest(bns []uint64) ([]*types.Block, s blocks, stid, err := sh.syncProtocol.GetBlocksByNumber(ctx, bns) if err != nil { - sh.logger.Warn().Err(err).Str("stream", string(stid)).Msg(WrapStagedSyncMsg("failed to doGetBlockHashesRequest")) + sh.logger.Warn().Err(err). + Str("stream", string(stid)). + Msg(WrapStagedSyncMsg("failed to doGetBlockHashesRequest")) return nil, stid, err } return blocks, stid, nil diff --git a/api/service/stagedstreamsync/stage.go b/api/service/stagedstreamsync/stage.go index 832cf4e3c7..255560a0ff 100644 --- a/api/service/stagedstreamsync/stage.go +++ b/api/service/stagedstreamsync/stage.go @@ -2,7 +2,6 @@ package stagedstreamsync import ( "context" - "errors" "github.com/ethereum/go-ethereum/common" sttypes "github.com/harmony-one/harmony/p2p/stream/types" @@ -48,9 +47,6 @@ type Stage struct { Disabled bool } -var ErrStopped = errors.New("stopped") -var ErrRevert = errors.New("unwound") - // StageState is the state of the stage. type StageState struct { state *StagedStreamSync diff --git a/api/service/stagedstreamsync/stage_bodies.go b/api/service/stagedstreamsync/stage_bodies.go index 1f122b0d9c..1fbfcbc2e6 100644 --- a/api/service/stagedstreamsync/stage_bodies.go +++ b/api/service/stagedstreamsync/stage_bodies.go @@ -170,7 +170,7 @@ func (b *StageBodies) runBlockWorkerLoop(gbm *blockDownloadManager, wg *sync.Wai gbm.HandleRequestError(batch, err, stid) } else { if err = b.saveBlocks(gbm.tx, batch, blockBytes, sigBytes, loopID, stid); err != nil { - panic("[STAGED_STREAM_SYNC] saving downloaded blocks to db failed.") + panic(ErrSaveBlocksToDbFailed) } gbm.HandleRequestResult(batch, blockBytes, sigBytes, loopID, stid) if b.configs.logProgress { diff --git a/api/service/stagedstreamsync/syncing.go b/api/service/stagedstreamsync/syncing.go index 046ba6843b..b793151c63 100644 --- a/api/service/stagedstreamsync/syncing.go +++ b/api/service/stagedstreamsync/syncing.go @@ -314,7 +314,7 @@ func (s *StagedStreamSync) estimateCurrentNumber() (uint64, error) { return 0, s.ctx.Err() default: } - return 0, errors.New("zero block number response from remote nodes") + return 0, ErrZeroBlockResponse } bn := computeBlockNumberByMaxVote(cnResults) return bn, nil diff --git a/api/service/stagedstreamsync/types.go b/api/service/stagedstreamsync/types.go index 00cfba79a2..6d6326452e 100644 --- a/api/service/stagedstreamsync/types.go +++ b/api/service/stagedstreamsync/types.go @@ -270,7 +270,7 @@ func (q priorityQueue) Swap(i, j int) { func (q *priorityQueue) Push(x interface{}) { item, ok := x.(bnPrioritizedItem) if !ok { - panic("wrong type of getBlockNumber interface") + panic(ErrWrongGetBlockNumberType) } *q = append(*q, item) } diff --git a/hmy/downloader/metric.go b/hmy/downloader/metric.go index 9338fd1304..2995db7b31 100644 --- a/hmy/downloader/metric.go +++ b/hmy/downloader/metric.go @@ -23,7 +23,7 @@ var ( consensusTriggeredDownloadCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "consensus_trigger", Help: "number of times consensus triggered download task", }, @@ -33,7 +33,7 @@ var ( longRangeSyncedBlockCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "num_blocks_synced_long_range", Help: "number of blocks synced in long range sync", }, @@ -43,7 +43,7 @@ var ( longRangeFailInsertedBlockCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "num_blocks_failed_long_range", Help: "number of blocks failed to insert into change in long range sync", }, @@ -53,7 +53,7 @@ var ( numShortRangeCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "num_short_range", Help: "number of short range sync is triggered", }, @@ -63,7 +63,7 @@ var ( numFailedDownloadCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "failed_download", Help: "number of downloading is failed", }, @@ -73,7 +73,7 @@ var ( numBlocksInsertedShortRangeHistogramVec = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "num_blocks_inserted_short_range", Help: "number of blocks inserted for each short range sync", // Buckets: 0, 1, 2, 4, +INF (capped at 10) @@ -85,7 +85,7 @@ var ( numBlocksInsertedBeaconHelperCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "num_blocks_inserted_beacon_helper", Help: "number of blocks inserted from beacon helper", }, From 35d393cc795c6023e3a8ec4bc056f8243ba2f8ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Mon, 2 Jan 2023 22:48:42 +0800 Subject: [PATCH 035/420] refactor p2p host creation --- p2p/host.go | 56 +++++++++++++++++-- .../common/streammanager/streammanager.go | 10 +++- p2p/stream/types/utils.go | 4 +- 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/p2p/host.go b/p2p/host.go index a2326c8125..f64a0cd624 100644 --- a/p2p/host.go +++ b/p2p/host.go @@ -12,6 +12,9 @@ import ( "time" "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/routing" + dht "github.com/libp2p/go-libp2p-kad-dht" libp2p_pubsub "github.com/libp2p/go-libp2p-pubsub" libp2p_crypto "github.com/libp2p/go-libp2p/core/crypto" libp2p_host "github.com/libp2p/go-libp2p/core/host" @@ -19,6 +22,10 @@ import ( libp2p_peer "github.com/libp2p/go-libp2p/core/peer" libp2p_peerstore "github.com/libp2p/go-libp2p/core/peerstore" "github.com/libp2p/go-libp2p/core/protocol" + "github.com/libp2p/go-libp2p/p2p/net/connmgr" + + "github.com/libp2p/go-libp2p/p2p/security/noise" + libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls" ma "github.com/multiformats/go-multiaddr" "github.com/pkg/errors" "github.com/rs/zerolog" @@ -111,19 +118,56 @@ func NewHost(cfg HostConfig) (Host, error) { self = cfg.Self key = cfg.BLSKey ) - listenAddr, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%s", self.IP, self.Port)) + // listenAddr, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%s", self.IP, self.Port)) + // if err != nil { + // return nil, errors.Wrapf(err, + // "cannot create listen multiaddr from port %#v", self.Port) + // } + + addr := fmt.Sprintf("/ip4/%s/tcp/%s", self.IP, self.Port) + listenAddr := libp2p.ListenAddrStrings( + addr, // regular tcp connections + addr+"/quic", // a UDP endpoint for the QUIC transport + ) + + ctx, cancel := context.WithCancel(context.Background()) + connmgr, err := connmgr.NewConnManager( + int(cfg.MaxPeers), // Lowwater + int(cfg.MaxPeers)*cfg.MaxConnPerIP, // HighWater, + connmgr.WithGracePeriod(time.Minute), + ) if err != nil { - return nil, errors.Wrapf(err, - "cannot create listen multiaddr from port %#v", self.Port) + cancel() + return nil, err } + var idht *dht.IpfsDHT - ctx, cancel := context.WithCancel(context.Background()) p2pHost, err := libp2p.New( - libp2p.ListenAddrs(listenAddr), + listenAddr, libp2p.Identity(key), + // support TLS connections + libp2p.Security(libp2ptls.ID, libp2ptls.New), + // support noise connections + libp2p.Security(noise.ID, noise.New), + // support any other default transports (TCP) + libp2p.DefaultTransports, + // Let's prevent our peer from having too many + // connections by attaching a connection manager. + libp2p.ConnectionManager(connmgr), + // Attempt to open ports using uPNP for NATed hosts. + libp2p.NATPortMap(), + libp2p.Routing(func(h host.Host) (routing.PeerRouting, error) { + idht, err = dht.New(ctx, h) + return idht, err + }), + // to help other peers to figure out if they are behind + // NATs, launch the server-side of AutoNAT too (AutoRelay + // already runs the client) + // This service is highly rate-limited and should not cause any + // performance issues. libp2p.EnableNATService(), - libp2p.ForceReachabilityPublic(), libp2p.BandwidthReporter(newCounter()), + libp2p.ForceReachabilityPublic(), // prevent dialing of public addresses libp2p.ConnectionGater(NewGater(cfg.DisablePrivateIPScan)), ) diff --git a/p2p/stream/common/streammanager/streammanager.go b/p2p/stream/common/streammanager/streammanager.go index a341184981..fe4505029c 100644 --- a/p2p/stream/common/streammanager/streammanager.go +++ b/p2p/stream/common/streammanager/streammanager.go @@ -419,9 +419,13 @@ func (ss *streamSet) addStream(st sttypes.Stream) { ss.lock.Lock() defer ss.lock.Unlock() - ss.streams[st.ID()] = st - spec, _ := st.ProtoSpec() - ss.numByProto[spec]++ + if spec, err := st.ProtoSpec(); err != nil { + return + } else { + ss.streams[st.ID()] = st + ss.numByProto[spec]++ + } + } func (ss *streamSet) deleteStream(st sttypes.Stream) { diff --git a/p2p/stream/types/utils.go b/p2p/stream/types/utils.go index 86eb882183..c27d95d60f 100644 --- a/p2p/stream/types/utils.go +++ b/p2p/stream/types/utils.go @@ -28,6 +28,7 @@ const ( // ProtoID is the protocol id for streaming, an alias of libp2p stream protocol ID。 // The stream protocol ID is composed of following components: +// ex: harmony/sync/partner/0/1.0.0/1 // 1. Service - Currently, only sync service is supported. // 2. NetworkType - mainnet, testnet, stn, e.t.c. // 3. ShardID - shard ID of the current protocol. @@ -37,8 +38,7 @@ type ProtoID libp2p_proto.ID // ProtoSpec is the un-serialized stream proto id specification // TODO: move this to service wise module since different protocol might have different -// -// protoID information +// protoID information type ProtoSpec struct { Service string NetworkType nodeconfig.NetworkType From 601422a63140135e34a3aa7a8c6c1edcbdaa23b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Tue, 3 Jan 2023 21:35:04 +0800 Subject: [PATCH 036/420] fix initsync and host creation --- api/service/stagedstreamsync/downloader.go | 15 ++++++++------- api/service/stagedstreamsync/stage_heads.go | 2 -- p2p/host.go | 16 +++++++--------- p2p/stream/protocols/sync/protocol.go | 2 +- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go index bb3b8c82eb..28c43c3e87 100644 --- a/api/service/stagedstreamsync/downloader.go +++ b/api/service/stagedstreamsync/downloader.go @@ -186,7 +186,7 @@ func (d *Downloader) waitForBootFinish() { func (d *Downloader) loop() { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() - initSync := d.bc.ShardID() != shard.BeaconChainShardID + initSync := true trigger := func() { select { case d.downloadC <- struct{}{}: @@ -228,11 +228,13 @@ func (d *Downloader) loop() { time.Sleep(1 * time.Second) continue } - d.logger.Info().Int("block added", addedBN). - Uint64("current height", d.bc.CurrentBlock().NumberU64()). - Bool("initSync", initSync). - Uint32("shard", d.bc.ShardID()). - Msg(WrapStagedSyncMsg("sync finished")) + if initSync { + d.logger.Info().Int("block added", addedBN). + Uint64("current height", d.bc.CurrentBlock().NumberU64()). + Bool("initSync", initSync). + Uint32("shard", d.bc.ShardID()). + Msg(WrapStagedSyncMsg("sync finished")) + } if addedBN != 0 { // If block number has been changed, trigger another sync @@ -242,7 +244,6 @@ func (d *Downloader) loop() { d.bh.insertSync() } } - d.stagedSyncInstance.initSync = false initSync = false case <-d.closeC: diff --git a/api/service/stagedstreamsync/stage_heads.go b/api/service/stagedstreamsync/stage_heads.go index 1ddd285aa0..8e1531a5ee 100644 --- a/api/service/stagedstreamsync/stage_heads.go +++ b/api/service/stagedstreamsync/stage_heads.go @@ -73,8 +73,6 @@ func (heads *StageHeads) Exec(firstCycle bool, invalidBlockRevert bool, s *Stage } if currentHeight >= maxHeight { - utils.Logger().Info().Uint64("current number", currentHeight).Uint64("target number", maxHeight). - Msg(WrapStagedSyncMsg("early return of long range sync")) return nil } diff --git a/p2p/host.go b/p2p/host.go index f64a0cd624..a2bd5996b8 100644 --- a/p2p/host.go +++ b/p2p/host.go @@ -118,11 +118,6 @@ func NewHost(cfg HostConfig) (Host, error) { self = cfg.Self key = cfg.BLSKey ) - // listenAddr, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%s", self.IP, self.Port)) - // if err != nil { - // return nil, errors.Wrapf(err, - // "cannot create listen multiaddr from port %#v", self.Port) - // } addr := fmt.Sprintf("/ip4/%s/tcp/%s", self.IP, self.Port) listenAddr := libp2p.ListenAddrStrings( @@ -131,9 +126,10 @@ func NewHost(cfg HostConfig) (Host, error) { ) ctx, cancel := context.WithCancel(context.Background()) + // TODO: move low and high to configs connmgr, err := connmgr.NewConnManager( - int(cfg.MaxPeers), // Lowwater - int(cfg.MaxPeers)*cfg.MaxConnPerIP, // HighWater, + int(10), // Lowwater + int(10000)*cfg.MaxConnPerIP, // HighWater, connmgr.WithGracePeriod(time.Minute), ) if err != nil { @@ -141,7 +137,6 @@ func NewHost(cfg HostConfig) (Host, error) { return nil, err } var idht *dht.IpfsDHT - p2pHost, err := libp2p.New( listenAddr, libp2p.Identity(key), @@ -167,7 +162,10 @@ func NewHost(cfg HostConfig) (Host, error) { // performance issues. libp2p.EnableNATService(), libp2p.BandwidthReporter(newCounter()), - libp2p.ForceReachabilityPublic(), + // ForceReachabilityPublic overrides automatic reachability detection in the AutoNAT subsystem, + // forcing the local node to believe it is reachable externally. + // libp2p.ForceReachabilityPublic(), + // prevent dialing of public addresses libp2p.ConnectionGater(NewGater(cfg.DisablePrivateIPScan)), ) diff --git a/p2p/stream/protocols/sync/protocol.go b/p2p/stream/protocols/sync/protocol.go index c63e6c45a5..68f92bb8e1 100644 --- a/p2p/stream/protocols/sync/protocol.go +++ b/p2p/stream/protocols/sync/protocol.go @@ -171,7 +171,7 @@ func (p *Protocol) HandleStream(raw libp2p_network.Stream) { Msg("failed to add new stream") return } - fmt.Println("Node connected to", raw.Conn().RemotePeer().String(), "(", st.ProtoID(), ")") + fmt.Println("Connected to", raw.Conn().RemotePeer().String(), "(", st.ProtoID(), ")") st.run() } From e5f18d3f368eead9e8adcb0d5f65c9f96d71d93a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Wed, 4 Jan 2023 14:34:09 +0800 Subject: [PATCH 037/420] fix short range hash chain --- api/service/stagedstreamsync/short_range_helper.go | 10 +++------- api/service/stagedstreamsync/stage_short_range.go | 8 +------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/api/service/stagedstreamsync/short_range_helper.go b/api/service/stagedstreamsync/short_range_helper.go index 210d1aa0c8..90415c87c3 100644 --- a/api/service/stagedstreamsync/short_range_helper.go +++ b/api/service/stagedstreamsync/short_range_helper.go @@ -139,15 +139,11 @@ func (sh *srHelper) checkPrerequisites() error { return nil } -func (sh *srHelper) prepareBlockHashNumbers(curNumber uint64, count int) []uint64 { +func (sh *srHelper) prepareBlockHashNumbers(curNumber uint64) []uint64 { - n := count - if count > BlockHashesPerRequest { - n = BlockHashesPerRequest - } - res := make([]uint64, 0, n) + res := make([]uint64, 0, BlockHashesPerRequest) - for bn := curNumber + 1; bn <= curNumber+uint64(n); bn++ { + for bn := curNumber + 1; bn <= curNumber+uint64(BlockHashesPerRequest); bn++ { res = append(res, bn) } return res diff --git a/api/service/stagedstreamsync/stage_short_range.go b/api/service/stagedstreamsync/stage_short_range.go index de83beafa3..cfe29f39e1 100644 --- a/api/service/stagedstreamsync/stage_short_range.go +++ b/api/service/stagedstreamsync/stage_short_range.go @@ -54,11 +54,6 @@ func (sr *StageShortRange) Exec(firstCycle bool, invalidBlockRevert bool, s *Sta return nil } - curBN := sr.configs.bc.CurrentBlock().NumberU64() - if curBN >= s.state.status.targetBN { - return nil - } - // do short range sync n, err := sr.doShortRangeSync(s) s.state.inserted = n @@ -109,8 +104,7 @@ func (sr *StageShortRange) doShortRangeSync(s *StageState) (int, error) { return 0, errors.Wrap(err, "prerequisite") } curBN := sr.configs.bc.CurrentBlock().NumberU64() - blkCount := int(s.state.status.targetBN) - int(curBN) - blkNums := sh.prepareBlockHashNumbers(curBN, blkCount) + blkNums := sh.prepareBlockHashNumbers(curBN) hashChain, whitelist, err := sh.getHashChain(blkNums) if err != nil { return 0, errors.Wrap(err, "getHashChain") From cc93332b5da05f1aca7109fb8624c22df1f034db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Thu, 5 Jan 2023 15:34:43 +0800 Subject: [PATCH 038/420] fix beacon node detection for p2p protocol --- api/service/stagedstreamsync/downloader.go | 13 +++++++------ api/service/stagedstreamsync/downloaders.go | 4 ++-- hmy/downloader/downloader.go | 13 +++++++------ hmy/downloader/downloaders.go | 4 ++-- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go index 28c43c3e87..a7b7402c21 100644 --- a/api/service/stagedstreamsync/downloader.go +++ b/api/service/stagedstreamsync/downloader.go @@ -36,15 +36,16 @@ type ( ) // NewDownloader creates a new downloader -func NewDownloader(host p2p.Host, bc core.BlockChain, config Config) *Downloader { +func NewDownloader(host p2p.Host, bc core.BlockChain, isBeaconNode bool, config Config) *Downloader { config.fixValues() sp := sync.NewProtocol(sync.Config{ - Chain: bc, - Host: host.GetP2PHost(), - Discovery: host.GetDiscovery(), - ShardID: nodeconfig.ShardID(bc.ShardID()), - Network: config.Network, + Chain: bc, + Host: host.GetP2PHost(), + Discovery: host.GetDiscovery(), + ShardID: nodeconfig.ShardID(bc.ShardID()), + Network: config.Network, + BeaconNode: isBeaconNode, SmSoftLowCap: config.SmSoftLowCap, SmHardLowCap: config.SmHardLowCap, diff --git a/api/service/stagedstreamsync/downloaders.go b/api/service/stagedstreamsync/downloaders.go index 7b5881f100..0e79c79631 100644 --- a/api/service/stagedstreamsync/downloaders.go +++ b/api/service/stagedstreamsync/downloaders.go @@ -17,7 +17,7 @@ type Downloaders struct { // NewDownloaders creates Downloaders for sync of multiple blockchains func NewDownloaders(host p2p.Host, bcs []core.BlockChain, config Config) *Downloaders { ds := make(map[uint32]*Downloader) - + isBeaconNode := len(bcs) == 1 for _, bc := range bcs { if bc == nil { continue @@ -25,7 +25,7 @@ func NewDownloaders(host p2p.Host, bcs []core.BlockChain, config Config) *Downlo if _, ok := ds[bc.ShardID()]; ok { continue } - ds[bc.ShardID()] = NewDownloader(host, bc, config) + ds[bc.ShardID()] = NewDownloader(host, bc, isBeaconNode, config) } return &Downloaders{ ds: ds, diff --git a/hmy/downloader/downloader.go b/hmy/downloader/downloader.go index f5ab8580b7..01ec242abb 100644 --- a/hmy/downloader/downloader.go +++ b/hmy/downloader/downloader.go @@ -44,15 +44,16 @@ type ( ) // NewDownloader creates a new downloader -func NewDownloader(host p2p.Host, bc core.BlockChain, config Config) *Downloader { +func NewDownloader(host p2p.Host, bc core.BlockChain, isBeaconNode bool, config Config) *Downloader { config.fixValues() sp := sync.NewProtocol(sync.Config{ - Chain: bc, - Host: host.GetP2PHost(), - Discovery: host.GetDiscovery(), - ShardID: nodeconfig.ShardID(bc.ShardID()), - Network: config.Network, + Chain: bc, + Host: host.GetP2PHost(), + Discovery: host.GetDiscovery(), + ShardID: nodeconfig.ShardID(bc.ShardID()), + Network: config.Network, + BeaconNode: isBeaconNode, SmSoftLowCap: config.SmSoftLowCap, SmHardLowCap: config.SmHardLowCap, diff --git a/hmy/downloader/downloaders.go b/hmy/downloader/downloaders.go index a1cbfa5a82..528e47e84f 100644 --- a/hmy/downloader/downloaders.go +++ b/hmy/downloader/downloaders.go @@ -17,7 +17,7 @@ type Downloaders struct { // NewDownloaders creates Downloaders for sync of multiple blockchains func NewDownloaders(host p2p.Host, bcs []core.BlockChain, config Config) *Downloaders { ds := make(map[uint32]*Downloader) - + isBeaconNode := len(bcs) == 1 for _, bc := range bcs { if bc == nil { continue @@ -25,7 +25,7 @@ func NewDownloaders(host p2p.Host, bcs []core.BlockChain, config Config) *Downlo if _, ok := ds[bc.ShardID()]; ok { continue } - ds[bc.ShardID()] = NewDownloader(host, bc, config) + ds[bc.ShardID()] = NewDownloader(host, bc, isBeaconNode, config) } return &Downloaders{ ds: ds, From a4a656a6d50cb2afe936a345977be22ccada9d5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Fri, 6 Jan 2023 09:02:40 +0800 Subject: [PATCH 039/420] refactor stream peer cooldown and fix protocol beacon node field --- p2p/stream/common/streammanager/cooldown.go | 7 ++++++- .../common/streammanager/streammanager.go | 4 ++++ p2p/stream/protocols/sync/protocol.go | 20 +++++++------------ 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/p2p/stream/common/streammanager/cooldown.go b/p2p/stream/common/streammanager/cooldown.go index d8b58346c0..f22f4956f0 100644 --- a/p2p/stream/common/streammanager/cooldown.go +++ b/p2p/stream/common/streammanager/cooldown.go @@ -30,11 +30,16 @@ func newCoolDownCache() *coolDownCache { func (cache *coolDownCache) Has(id peer.ID) bool { cache.mu.Lock() defer cache.mu.Unlock() + has := cache.timeCache.Has(string(id)) + return has +} + +// Add adds the peer ID to the cache +func (cache *coolDownCache) Add(id peer.ID) { has := cache.timeCache.Has(string(id)) if !has { cache.timeCache.Add(string(id)) } - return has } // Reset the cool down cache diff --git a/p2p/stream/common/streammanager/streammanager.go b/p2p/stream/common/streammanager/streammanager.go index fe4505029c..60d3a55213 100644 --- a/p2p/stream/common/streammanager/streammanager.go +++ b/p2p/stream/common/streammanager/streammanager.go @@ -315,12 +315,16 @@ func (sm *streamManager) discoverAndSetupStream(discCtx context.Context) (int, e // If the peer has the same ID and was just connected, skip. continue } + if _, ok := sm.streams.get(sttypes.StreamID(peer.ID)); ok { + continue + } discoveredPeersCounterVec.With(prometheus.Labels{"topic": string(sm.myProtoID)}).Inc() connecting += 1 go func(pid libp2p_peer.ID) { // The ctx here is using the module context instead of discover context err := sm.setupStreamWithPeer(sm.ctx, pid) if err != nil { + sm.coolDownCache.Add(peer.ID) sm.logger.Warn().Err(err).Str("peerID", string(pid)).Msg("failed to setup stream with peer") return } diff --git a/p2p/stream/protocols/sync/protocol.go b/p2p/stream/protocols/sync/protocol.go index 68f92bb8e1..7b5a1ff2d2 100644 --- a/p2p/stream/protocols/sync/protocol.go +++ b/p2p/stream/protocols/sync/protocol.go @@ -8,7 +8,6 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/harmony-one/harmony/consensus/engine" - "github.com/harmony-one/harmony/core" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" "github.com/harmony-one/harmony/internal/utils" @@ -17,7 +16,6 @@ import ( "github.com/harmony-one/harmony/p2p/stream/common/requestmanager" "github.com/harmony-one/harmony/p2p/stream/common/streammanager" sttypes "github.com/harmony-one/harmony/p2p/stream/types" - "github.com/harmony-one/harmony/shard" "github.com/hashicorp/go-version" libp2p_host "github.com/libp2p/go-libp2p/core/host" libp2p_network "github.com/libp2p/go-libp2p/core/network" @@ -60,12 +58,12 @@ type ( // Config is the sync protocol config Config struct { - Chain engine.ChainReader - Host libp2p_host.Host - Discovery discovery.Discovery - ShardID nodeconfig.ShardID - Network nodeconfig.NetworkType - + Chain engine.ChainReader + Host libp2p_host.Host + Discovery discovery.Discovery + ShardID nodeconfig.ShardID + Network nodeconfig.NetworkType + BeaconNode bool // stream manager config SmSoftLowCap int SmHardLowCap int @@ -78,13 +76,9 @@ type ( func NewProtocol(config Config) *Protocol { ctx, cancel := context.WithCancel(context.Background()) - isBeaconNode := config.Chain.ShardID() == shard.BeaconChainShardID - if _, ok := config.Chain.(*core.EpochChain); ok { - isBeaconNode = false - } sp := &Protocol{ chain: config.Chain, - beaconNode: isBeaconNode, + beaconNode: config.BeaconNode, disc: config.Discovery, config: config, ctx: ctx, From fecd4fdda3a8e5dfc32e57776b0a0a643fbec58e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Wed, 11 Jan 2023 22:55:15 +0800 Subject: [PATCH 040/420] refactor p2p host and routing --- api/service/stagedstreamsync/beacon_helper.go | 14 ++++---- api/service/stagedstreamsync/downloader.go | 22 +++++++----- api/service/stagedstreamsync/stage_epoch.go | 2 +- .../stagedstreamsync/stage_short_range.go | 3 +- .../stagedstreamsync/staged_stream_sync.go | 29 +++++++++------- api/service/stagedstreamsync/syncing.go | 2 ++ cmd/harmony/main.go | 8 ++--- p2p/discovery/discovery.go | 14 ++------ p2p/discovery/option.go | 8 +++-- p2p/host.go | 31 ++++++++++++----- .../common/streammanager/streammanager.go | 9 +++-- p2p/stream/protocols/sync/protocol.go | 34 +++++++++++++++++-- p2p/stream/types/interface.go | 2 ++ 13 files changed, 115 insertions(+), 63 deletions(-) diff --git a/api/service/stagedstreamsync/beacon_helper.go b/api/service/stagedstreamsync/beacon_helper.go index 2f684e0e27..a996f368bf 100644 --- a/api/service/stagedstreamsync/beacon_helper.go +++ b/api/service/stagedstreamsync/beacon_helper.go @@ -76,17 +76,19 @@ func (bh *beaconHelper) loop() { case it := <-bh.insertC: inserted, bn, err := bh.insertLastMileBlocks() - numBlocksInsertedBeaconHelperCounter.Add(float64(inserted)) if err != nil { bh.logger.Error().Err(err). Msg(WrapStagedSyncMsg("insert last mile blocks error")) + close(it.doneC) continue } - bh.logger.Info().Int("inserted", inserted). - Uint64("end height", bn). - Uint32("shard", bh.bc.ShardID()). - Msg(WrapStagedSyncMsg("insert last mile blocks")) - + if inserted > 0 { + numBlocksInsertedBeaconHelperCounter.Add(float64(inserted)) + bh.logger.Info().Int("inserted", inserted). + Uint64("end height", bn). + Uint32("shard", bh.bc.ShardID()). + Msg(WrapStagedSyncMsg("insert last mile blocks")) + } close(it.doneC) case <-bh.closeC: diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go index a7b7402c21..f53a5c41a8 100644 --- a/api/service/stagedstreamsync/downloader.go +++ b/api/service/stagedstreamsync/downloader.go @@ -24,6 +24,7 @@ type ( syncProtocol syncProtocol bh *beaconHelper stagedSyncInstance *StagedStreamSync + isBeaconNode bool downloadC chan struct{} closeC chan struct{} @@ -67,7 +68,7 @@ func NewDownloader(host p2p.Host, bc core.BlockChain, isBeaconNode bool, config ctx, cancel := context.WithCancel(context.Background()) //TODO: use mem db should be in config file - stagedSyncInstance, err := CreateStagedSync(ctx, bc, false, sp, config, logger, config.LogProgress) + stagedSyncInstance, err := CreateStagedSync(ctx, bc, false, isBeaconNode, sp, config, logger, config.LogProgress) if err != nil { cancel() return nil @@ -78,6 +79,7 @@ func NewDownloader(host p2p.Host, bc core.BlockChain, isBeaconNode bool, config syncProtocol: sp, bh: bh, stagedSyncInstance: stagedSyncInstance, + isBeaconNode: isBeaconNode, downloadC: make(chan struct{}), closeC: make(chan struct{}), @@ -187,7 +189,11 @@ func (d *Downloader) waitForBootFinish() { func (d *Downloader) loop() { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() - initSync := true + // for shard chain and beacon chain node, first we start with initSync=true to + // make sure it goes through the long range sync first. + // for epoch chain we do only need to go through epoch sync process + initSync := d.isBeaconNode || d.bc.ShardID() != shard.BeaconChainShardID + trigger := func() { select { case d.downloadC <- struct{}{}: @@ -217,7 +223,7 @@ func (d *Downloader) loop() { } } - // If error happens, sleep 5 seconds and retry + // If any error happens, sleep 5 seconds and retry d.logger.Error(). Err(err). Bool("initSync", initSync). @@ -227,7 +233,7 @@ func (d *Downloader) loop() { trigger() }() time.Sleep(1 * time.Second) - continue + break } if initSync { d.logger.Info().Int("block added", addedBN). @@ -239,11 +245,11 @@ func (d *Downloader) loop() { if addedBN != 0 { // If block number has been changed, trigger another sync - // and try to add last mile from pub-sub (blocking) go trigger() - if d.bh != nil { - d.bh.insertSync() - } + } + // try to add last mile from pub-sub (blocking) + if d.bh != nil { + d.bh.insertSync() } initSync = false diff --git a/api/service/stagedstreamsync/stage_epoch.go b/api/service/stagedstreamsync/stage_epoch.go index 6320d82175..77dc57bfdd 100644 --- a/api/service/stagedstreamsync/stage_epoch.go +++ b/api/service/stagedstreamsync/stage_epoch.go @@ -49,7 +49,7 @@ func (sr *StageEpoch) Exec(firstCycle bool, invalidBlockRevert bool, s *StageSta return nil } - if _, ok := sr.configs.bc.(*core.EpochChain); !ok { + if sr.configs.bc.ShardID() != shard.BeaconChainShardID || s.state.isBeaconNode { return nil } diff --git a/api/service/stagedstreamsync/stage_short_range.go b/api/service/stagedstreamsync/stage_short_range.go index cfe29f39e1..75f51ee1e7 100644 --- a/api/service/stagedstreamsync/stage_short_range.go +++ b/api/service/stagedstreamsync/stage_short_range.go @@ -6,6 +6,7 @@ import ( "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/internal/utils" sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/harmony-one/harmony/shard" "github.com/ledgerwatch/erigon-lib/kv" "github.com/pkg/errors" ) @@ -50,7 +51,7 @@ func (sr *StageShortRange) Exec(firstCycle bool, invalidBlockRevert bool, s *Sta return nil } - if _, ok := sr.configs.bc.(*core.EpochChain); ok { + if sr.configs.bc.ShardID() == shard.BeaconChainShardID && !s.state.isBeaconNode { return nil } diff --git a/api/service/stagedstreamsync/staged_stream_sync.go b/api/service/stagedstreamsync/staged_stream_sync.go index 9603f5b06b..e73edd622f 100644 --- a/api/service/stagedstreamsync/staged_stream_sync.go +++ b/api/service/stagedstreamsync/staged_stream_sync.go @@ -54,19 +54,20 @@ func (ib *InvalidBlock) addBadStream(bsID sttypes.StreamID) { } type StagedStreamSync struct { - ctx context.Context - bc core.BlockChain - isBeacon bool - isExplorer bool - db kv.RwDB - protocol syncProtocol - gbm *blockDownloadManager // initialized when finished get block number - inserted int - config Config - logger zerolog.Logger - status *status //TODO: merge this with currentSyncCycle - initSync bool // if sets to true, node start long range syncing - UseMemDB bool + ctx context.Context + bc core.BlockChain + isBeacon bool + isExplorer bool + db kv.RwDB + protocol syncProtocol + isBeaconNode bool + gbm *blockDownloadManager // initialized when finished get block number + inserted int + config Config + logger zerolog.Logger + status *status //TODO: merge this with currentSyncCycle + initSync bool // if sets to true, node start long range syncing + UseMemDB bool revertPoint *uint64 // used to run stages prevRevertPoint *uint64 // used to get value from outside of staged sync after cycle (for example to notify RPCDaemon) @@ -254,6 +255,7 @@ func New(ctx context.Context, stagesList []*Stage, isBeacon bool, protocol syncProtocol, + isBeaconNode bool, useMemDB bool, config Config, logger zerolog.Logger, @@ -291,6 +293,7 @@ func New(ctx context.Context, isBeacon: isBeacon, db: db, protocol: protocol, + isBeaconNode: isBeaconNode, gbm: nil, status: &status, inserted: 0, diff --git a/api/service/stagedstreamsync/syncing.go b/api/service/stagedstreamsync/syncing.go index b793151c63..de9b884816 100644 --- a/api/service/stagedstreamsync/syncing.go +++ b/api/service/stagedstreamsync/syncing.go @@ -38,6 +38,7 @@ var Buckets = []string{ func CreateStagedSync(ctx context.Context, bc core.BlockChain, UseMemDB bool, + isBeaconNode bool, protocol syncProtocol, config Config, logger zerolog.Logger, @@ -87,6 +88,7 @@ func CreateStagedSync(ctx context.Context, stages, isBeacon, protocol, + isBeaconNode, UseMemDB, config, logger, diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 6c83636673..6ab3ed1024 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -878,8 +878,8 @@ func setupPrometheusService(node *node.Node, hc harmonyconfig.HarmonyConfig, sid func setupSyncService(node *node.Node, host p2p.Host, hc harmonyconfig.HarmonyConfig) { blockchains := []core.BlockChain{node.Blockchain()} - if !node.IsRunningBeaconChain() { - blockchains = append(blockchains, node.Beaconchain()) + if node.Blockchain().ShardID() != shard.BeaconChainShardID { + blockchains = append(blockchains, node.EpochChain()) } dConfig := downloader.Config{ @@ -913,8 +913,8 @@ func setupSyncService(node *node.Node, host p2p.Host, hc harmonyconfig.HarmonyCo func setupStagedSyncService(node *node.Node, host p2p.Host, hc harmonyconfig.HarmonyConfig) { blockchains := []core.BlockChain{node.Blockchain()} - if !node.IsRunningBeaconChain() { - blockchains = append(blockchains, node.Beaconchain()) + if node.Blockchain().ShardID() != shard.BeaconChainShardID { + blockchains = append(blockchains, node.EpochChain()) } sConfig := stagedstreamsync.Config{ diff --git a/p2p/discovery/discovery.go b/p2p/discovery/discovery.go index fb9591c267..53372edd67 100644 --- a/p2p/discovery/discovery.go +++ b/p2p/discovery/discovery.go @@ -5,6 +5,7 @@ import ( "time" "github.com/harmony-one/harmony/internal/utils" + dht "github.com/libp2p/go-libp2p-kad-dht" libp2p_dht "github.com/libp2p/go-libp2p-kad-dht" "github.com/libp2p/go-libp2p/core/discovery" libp2p_host "github.com/libp2p/go-libp2p/core/host" @@ -37,19 +38,8 @@ type dhtDiscovery struct { } // NewDHTDiscovery creates a new dhtDiscovery that implements Discovery interface. -func NewDHTDiscovery(host libp2p_host.Host, opt DHTConfig) (Discovery, error) { - opts, err := opt.getLibp2pRawOptions() - if err != nil { - return nil, err - } - ctx, cancel := context.WithCancel(context.Background()) - dht, err := libp2p_dht.New(ctx, host, opts...) - if err != nil { - cancel() - return nil, err - } +func NewDHTDiscovery(ctx context.Context, cancel context.CancelFunc, host libp2p_host.Host, dht *dht.IpfsDHT, opt DHTConfig) (Discovery, error) { d := libp2p_dis.NewRoutingDiscovery(dht) - logger := utils.Logger().With().Str("module", "discovery").Logger() return &dhtDiscovery{ dht: dht, diff --git a/p2p/discovery/option.go b/p2p/discovery/option.go index 0afe6b8a2f..ce42592703 100644 --- a/p2p/discovery/option.go +++ b/p2p/discovery/option.go @@ -5,6 +5,7 @@ import ( p2ptypes "github.com/harmony-one/harmony/p2p/types" badger "github.com/ipfs/go-ds-badger" + dht "github.com/libp2p/go-libp2p-kad-dht" libp2p_dht "github.com/libp2p/go-libp2p-kad-dht" ) @@ -14,10 +15,11 @@ type DHTConfig struct { BootNodes []string DataStoreFile *string // File path to store DHT data. Shall be only used for bootstrap nodes. DiscConcurrency int + DHT *dht.IpfsDHT } -// getLibp2pRawOptions get the raw libp2p options as a slice. -func (opt DHTConfig) getLibp2pRawOptions() ([]libp2p_dht.Option, error) { +// GetLibp2pRawOptions get the raw libp2p options as a slice. +func (opt DHTConfig) GetLibp2pRawOptions() ([]libp2p_dht.Option, error) { var opts []libp2p_dht.Option bootOption, err := getBootstrapOption(opt.BootNodes) @@ -40,6 +42,8 @@ func (opt DHTConfig) getLibp2pRawOptions() ([]libp2p_dht.Option, error) { opts = append(opts, libp2p_dht.Concurrency(opt.DiscConcurrency)) } + opts = append(opts, libp2p_dht.DisableAutoRefresh()) + return opts, nil } diff --git a/p2p/host.go b/p2p/host.go index a2bd5996b8..fcd2ea3b4d 100644 --- a/p2p/host.go +++ b/p2p/host.go @@ -128,7 +128,7 @@ func NewHost(cfg HostConfig) (Host, error) { ctx, cancel := context.WithCancel(context.Background()) // TODO: move low and high to configs connmgr, err := connmgr.NewConnManager( - int(10), // Lowwater + int(10), // LowWater int(10000)*cfg.MaxConnPerIP, // HighWater, connmgr.WithGracePeriod(time.Minute), ) @@ -137,6 +137,7 @@ func NewHost(cfg HostConfig) (Host, error) { return nil, err } var idht *dht.IpfsDHT + var opt discovery.DHTConfig p2pHost, err := libp2p.New( listenAddr, libp2p.Identity(key), @@ -152,33 +153,41 @@ func NewHost(cfg HostConfig) (Host, error) { // Attempt to open ports using uPNP for NATed hosts. libp2p.NATPortMap(), libp2p.Routing(func(h host.Host) (routing.PeerRouting, error) { - idht, err = dht.New(ctx, h) + opt = discovery.DHTConfig{ + BootNodes: cfg.BootNodes, + DataStoreFile: cfg.DataStoreFile, + DiscConcurrency: cfg.DiscConcurrency, + } + opts, err := opt.GetLibp2pRawOptions() + if err != nil { + return nil, err + } + idht, err = dht.New(ctx, h, opts...) return idht, err }), + // to help other peers to figure out if they are behind // NATs, launch the server-side of AutoNAT too (AutoRelay // already runs the client) // This service is highly rate-limited and should not cause any // performance issues. libp2p.EnableNATService(), + // Bandwidth Reporter libp2p.BandwidthReporter(newCounter()), // ForceReachabilityPublic overrides automatic reachability detection in the AutoNAT subsystem, // forcing the local node to believe it is reachable externally. // libp2p.ForceReachabilityPublic(), - + // libp2p.DisableRelay(), + libp2p.EnableRelayService(), // prevent dialing of public addresses - libp2p.ConnectionGater(NewGater(cfg.DisablePrivateIPScan)), + // libp2p.ConnectionGater(NewGater(cfg.DisablePrivateIPScan)), ) if err != nil { cancel() return nil, errors.Wrapf(err, "cannot initialize libp2p host") } - disc, err := discovery.NewDHTDiscovery(p2pHost, discovery.DHTConfig{ - BootNodes: cfg.BootNodes, - DataStoreFile: cfg.DataStoreFile, - DiscConcurrency: cfg.DiscConcurrency, - }) + disc, err := discovery.NewDHTDiscovery(ctx, cancel, p2pHost, idht, opt) if err != nil { cancel() return nil, errors.Wrap(err, "cannot create DHT discovery") @@ -319,6 +328,10 @@ func (host *HostV2) AddStreamProtocol(protocols ...sttypes.Protocol) { for _, proto := range protocols { host.streamProtos = append(host.streamProtos, proto) host.h.SetStreamHandlerMatch(protocol.ID(proto.ProtoID()), proto.Match, proto.HandleStream) + // TODO: do we need to add handler match for shard proto id? + // if proto.IsBeaconNode() { + // host.h.SetStreamHandlerMatch(protocol.ID(proto.ShardProtoID()), proto.Match, proto.HandleStream) + // } } } diff --git a/p2p/stream/common/streammanager/streammanager.go b/p2p/stream/common/streammanager/streammanager.go index 60d3a55213..064cb0c2d0 100644 --- a/p2p/stream/common/streammanager/streammanager.go +++ b/p2p/stream/common/streammanager/streammanager.go @@ -73,7 +73,6 @@ func newStreamManager(pid sttypes.ProtoID, host host, pf peerFinder, handleStrea Str("protocol ID", string(pid)).Logger() protoSpec, _ := sttypes.ProtoIDToProtoSpec(pid) - fmt.Println("my peer id: ", host.ID().String()) fmt.Println("my proto id: ", pid) @@ -238,9 +237,6 @@ func (sm *streamManager) sanityCheckStream(st sttypes.Stream) error { if mySpec.ShardID != rmSpec.ShardID { return fmt.Errorf("unexpected shard ID: %v/%v", rmSpec.ShardID, mySpec.ShardID) } - if mySpec.ShardID == shard.BeaconChainShardID && !rmSpec.BeaconNode { - return fmt.Errorf("unexpected beacon node with shard ID: %v/%v", rmSpec.ShardID, mySpec.ShardID) - } return nil } @@ -311,7 +307,10 @@ func (sm *streamManager) discoverAndSetupStream(discCtx context.Context) (int, e connecting := 0 for peer := range peers { - if peer.ID == sm.host.ID() || sm.coolDownCache.Has(peer.ID) { + if peer.ID == sm.host.ID() { + continue + } + if sm.coolDownCache.Has(peer.ID) { // If the peer has the same ID and was just connected, skip. continue } diff --git a/p2p/stream/protocols/sync/protocol.go b/p2p/stream/protocols/sync/protocol.go index 7b5a1ff2d2..2a984a5818 100644 --- a/p2p/stream/protocols/sync/protocol.go +++ b/p2p/stream/protocols/sync/protocol.go @@ -16,6 +16,7 @@ import ( "github.com/harmony-one/harmony/p2p/stream/common/requestmanager" "github.com/harmony-one/harmony/p2p/stream/common/streammanager" sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/harmony-one/harmony/shard" "github.com/hashicorp/go-version" libp2p_host "github.com/libp2p/go-libp2p/core/host" libp2p_network "github.com/libp2p/go-libp2p/core/network" @@ -107,7 +108,10 @@ func (p *Protocol) Start() { p.sm.Start() p.rm.Start() p.rl.Start() - go p.advertiseLoop() + // If it's not EpochChain, advertise + if p.beaconNode || p.chain.ShardID() != shard.BeaconChainShardID { + go p.advertiseLoop() + } } // Close close the protocol @@ -129,11 +133,21 @@ func (p *Protocol) ProtoID() sttypes.ProtoID { return p.protoIDByVersion(MyVersion) } +// ShardProtoID returns the ProtoID of the sync protocol for shard nodes +func (p *Protocol) ShardProtoID() sttypes.ProtoID { + return p.protoIDByVersionForShardNodes(MyVersion) +} + // Version returns the sync protocol version func (p *Protocol) Version() *version.Version { return MyVersion } +// IsBeaconNode returns true if it is a beacon chain node +func (p *Protocol) IsBeaconNode() bool { + return p.beaconNode +} + // Match checks the compatibility to the target protocol ID. func (p *Protocol) Match(targetID string) bool { target, err := sttypes.ProtoIDToProtoSpec(sttypes.ProtoID(targetID)) @@ -173,9 +187,9 @@ func (p *Protocol) advertiseLoop() { for { sleep := p.advertise() select { - case <-time.After(sleep): case <-p.closeC: return + case <-time.After(sleep): } } } @@ -209,6 +223,11 @@ func (p *Protocol) supportedProtoIDs() []sttypes.ProtoID { pids := make([]sttypes.ProtoID, 0, len(vs)) for _, v := range vs { pids = append(pids, p.protoIDByVersion(v)) + // beacon node needs to inform shard nodes about it supports them as well for EpochChain + // basically beacon node can accept connection from shard nodes to share last epoch blocks + if p.beaconNode { + pids = append(pids, p.protoIDByVersionForShardNodes(v)) + } } return pids } @@ -228,6 +247,17 @@ func (p *Protocol) protoIDByVersion(v *version.Version) sttypes.ProtoID { return spec.ToProtoID() } +func (p *Protocol) protoIDByVersionForShardNodes(v *version.Version) sttypes.ProtoID { + spec := sttypes.ProtoSpec{ + Service: serviceSpecifier, + NetworkType: p.config.Network, + ShardID: p.config.ShardID, + Version: v, + BeaconNode: false, + } + return spec.ToProtoID() +} + // RemoveStream removes the stream of the given stream ID // TODO: add reason to parameters func (p *Protocol) RemoveStream(stID sttypes.StreamID) { diff --git a/p2p/stream/types/interface.go b/p2p/stream/types/interface.go index 424382cc8c..d7b60f78c3 100644 --- a/p2p/stream/types/interface.go +++ b/p2p/stream/types/interface.go @@ -13,6 +13,8 @@ type Protocol interface { Specifier() string Version() *version.Version ProtoID() ProtoID + // ShardProtoID() ProtoID + IsBeaconNode() bool Match(string) bool HandleStream(st libp2p_network.Stream) } From e49dfd7824c35c99ac92cfaa94ec020d424f02d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Thu, 12 Jan 2023 08:41:46 +0800 Subject: [PATCH 041/420] fix p2p discovery test issue --- p2p/discovery/discovery_test.go | 10 +++++++++- p2p/discovery/option_test.go | 6 +++--- p2p/stream/common/streammanager/streammanager.go | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/p2p/discovery/discovery_test.go b/p2p/discovery/discovery_test.go index 6ea902375f..f2c2f5b99e 100644 --- a/p2p/discovery/discovery_test.go +++ b/p2p/discovery/discovery_test.go @@ -3,9 +3,11 @@ package discovery // TODO: test this module import ( + "context" "testing" "github.com/libp2p/go-libp2p" + dht "github.com/libp2p/go-libp2p-kad-dht" ) func TestNewDHTDiscovery(t *testing.T) { @@ -13,7 +15,13 @@ func TestNewDHTDiscovery(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = NewDHTDiscovery(host, DHTConfig{}) + ctx, cancel := context.WithCancel(context.Background()) + var idht *dht.IpfsDHT + idht, err = dht.New(ctx, host) + if err != nil { + t.Fatal(err) + } + _, err = NewDHTDiscovery(ctx, cancel, host, idht, DHTConfig{}) if err != nil { t.Fatal(err) } diff --git a/p2p/discovery/option_test.go b/p2p/discovery/option_test.go index 747d7ca957..e782151f59 100644 --- a/p2p/discovery/option_test.go +++ b/p2p/discovery/option_test.go @@ -40,14 +40,14 @@ func TestDHTOption_getLibp2pRawOptions(t *testing.T) { opt: DHTConfig{ BootNodes: testAddrStr, }, - expLen: 1, + expLen: 2, }, { opt: DHTConfig{ BootNodes: testAddrStr, DataStoreFile: &validPath, }, - expLen: 2, + expLen: 3, }, { opt: DHTConfig{ @@ -58,7 +58,7 @@ func TestDHTOption_getLibp2pRawOptions(t *testing.T) { }, } for i, test := range tests { - opts, err := test.opt.getLibp2pRawOptions() + opts, err := test.opt.GetLibp2pRawOptions() if assErr := assertError(err, test.expErr); assErr != nil { t.Errorf("Test %v: %v", i, assErr) } diff --git a/p2p/stream/common/streammanager/streammanager.go b/p2p/stream/common/streammanager/streammanager.go index 064cb0c2d0..98e20cbaf9 100644 --- a/p2p/stream/common/streammanager/streammanager.go +++ b/p2p/stream/common/streammanager/streammanager.go @@ -323,7 +323,7 @@ func (sm *streamManager) discoverAndSetupStream(discCtx context.Context) (int, e // The ctx here is using the module context instead of discover context err := sm.setupStreamWithPeer(sm.ctx, pid) if err != nil { - sm.coolDownCache.Add(peer.ID) + sm.coolDownCache.Add(pid) sm.logger.Warn().Err(err).Str("peerID", string(pid)).Msg("failed to setup stream with peer") return } From bd0cf8c131a2281749c1d29c60c6c9b3995a9088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Fri, 13 Jan 2023 16:17:07 +0800 Subject: [PATCH 042/420] add MaxAdvertiseWaitTime to handle advertisements interval and address stream connection issue --- api/service/stagedstreamsync/const.go | 10 +- api/service/stagedstreamsync/downloader.go | 22 ++--- cmd/harmony/config.go | 2 + cmd/harmony/config_migrations.go | 8 ++ cmd/harmony/default.go | 109 ++++++++++++--------- cmd/harmony/flags.go | 9 ++ cmd/harmony/main.go | 21 ++-- internal/configs/harmony/harmony.go | 23 ++--- p2p/stream/protocols/sync/protocol.go | 17 ++-- rosetta/infra/harmony-mainnet.conf | 3 +- rosetta/infra/harmony-pstn.conf | 3 +- 11 files changed, 137 insertions(+), 90 deletions(-) diff --git a/api/service/stagedstreamsync/const.go b/api/service/stagedstreamsync/const.go index ed68d80b13..0e6bc6e2cf 100644 --- a/api/service/stagedstreamsync/const.go +++ b/api/service/stagedstreamsync/const.go @@ -36,11 +36,11 @@ type ( ServerOnly bool // parameters - Network nodeconfig.NetworkType - Concurrency int // Number of concurrent sync requests - MinStreams int // Minimum number of streams to do sync - InitStreams int // Number of streams requirement for initial bootstrap - + Network nodeconfig.NetworkType + Concurrency int // Number of concurrent sync requests + MinStreams int // Minimum number of streams to do sync + InitStreams int // Number of streams requirement for initial bootstrap + MaxAdvertiseWaitTime int // maximum time duration between protocol advertisements // stream manager config SmSoftLowCap int SmHardLowCap int diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go index f53a5c41a8..cfcd180c0f 100644 --- a/api/service/stagedstreamsync/downloader.go +++ b/api/service/stagedstreamsync/downloader.go @@ -41,17 +41,17 @@ func NewDownloader(host p2p.Host, bc core.BlockChain, isBeaconNode bool, config config.fixValues() sp := sync.NewProtocol(sync.Config{ - Chain: bc, - Host: host.GetP2PHost(), - Discovery: host.GetDiscovery(), - ShardID: nodeconfig.ShardID(bc.ShardID()), - Network: config.Network, - BeaconNode: isBeaconNode, - - SmSoftLowCap: config.SmSoftLowCap, - SmHardLowCap: config.SmHardLowCap, - SmHiCap: config.SmHiCap, - DiscBatch: config.SmDiscBatch, + Chain: bc, + Host: host.GetP2PHost(), + Discovery: host.GetDiscovery(), + ShardID: nodeconfig.ShardID(bc.ShardID()), + Network: config.Network, + BeaconNode: isBeaconNode, + MaxAdvertiseWaitTime: config.MaxAdvertiseWaitTime, + SmSoftLowCap: config.SmSoftLowCap, + SmHardLowCap: config.SmHardLowCap, + SmHiCap: config.SmHiCap, + DiscBatch: config.SmDiscBatch, }) host.AddStreamProtocol(sp) diff --git a/cmd/harmony/config.go b/cmd/harmony/config.go index dea5fb2e39..30c72c2e25 100644 --- a/cmd/harmony/config.go +++ b/cmd/harmony/config.go @@ -139,6 +139,8 @@ func getDefaultSyncConfig(nt nodeconfig.NetworkType) harmonyconfig.SyncConfig { return defaultTestNetSyncConfig case nodeconfig.Localnet: return defaultLocalNetSyncConfig + case nodeconfig.Partner: + return defaultPartnerSyncConfig default: return defaultElseSyncConfig } diff --git a/cmd/harmony/config_migrations.go b/cmd/harmony/config_migrations.go index eb98ccb794..3a256f49d2 100644 --- a/cmd/harmony/config_migrations.go +++ b/cmd/harmony/config_migrations.go @@ -318,6 +318,14 @@ func init() { return confTree } + migrations["2.5.10"] = func(confTree *toml.Tree) *toml.Tree { + if confTree.Get("Sync.MaxAdvertiseWaitTime") == nil { + confTree.Set("Sync.MaxAdvertiseWaitTime", defaultConfig.Sync.MaxAdvertiseWaitTime) + } + confTree.Set("Version", "2.5.11") + return confTree + } + // check that the latest version here is the same as in default.go largestKey := getNextVersion(migrations) if largestKey != tomlConfigVersion { diff --git a/cmd/harmony/default.go b/cmd/harmony/default.go index 7de12af9c2..7bdcc02d0d 100644 --- a/cmd/harmony/default.go +++ b/cmd/harmony/default.go @@ -5,7 +5,7 @@ import ( nodeconfig "github.com/harmony-one/harmony/internal/configs/node" ) -const tomlConfigVersion = "2.5.10" +const tomlConfigVersion = "2.5.11" const ( defNetworkType = nodeconfig.Mainnet @@ -159,59 +159,78 @@ var defaultStagedSyncConfig = harmonyconfig.StagedSyncConfig{ var ( defaultMainnetSyncConfig = harmonyconfig.SyncConfig{ - Enabled: false, - Downloader: false, - StagedSync: false, - StagedSyncCfg: defaultStagedSyncConfig, - Concurrency: 6, - MinPeers: 6, - InitStreams: 8, - DiscSoftLowCap: 8, - DiscHardLowCap: 6, - DiscHighCap: 128, - DiscBatch: 8, + Enabled: false, + Downloader: false, + StagedSync: false, + StagedSyncCfg: defaultStagedSyncConfig, + Concurrency: 6, + MinPeers: 6, + InitStreams: 8, + MaxAdvertiseWaitTime: 60, //minutes + DiscSoftLowCap: 8, + DiscHardLowCap: 6, + DiscHighCap: 128, + DiscBatch: 8, } defaultTestNetSyncConfig = harmonyconfig.SyncConfig{ - Enabled: true, - Downloader: false, - StagedSync: false, - StagedSyncCfg: defaultStagedSyncConfig, - Concurrency: 2, - MinPeers: 2, - InitStreams: 2, - DiscSoftLowCap: 2, - DiscHardLowCap: 2, - DiscHighCap: 1024, - DiscBatch: 3, + Enabled: true, + Downloader: false, + StagedSync: false, + StagedSyncCfg: defaultStagedSyncConfig, + Concurrency: 2, + MinPeers: 2, + InitStreams: 2, + MaxAdvertiseWaitTime: 5, //minutes + DiscSoftLowCap: 2, + DiscHardLowCap: 2, + DiscHighCap: 1024, + DiscBatch: 3, } defaultLocalNetSyncConfig = harmonyconfig.SyncConfig{ - Enabled: true, - Downloader: true, - StagedSync: false, - StagedSyncCfg: defaultStagedSyncConfig, - Concurrency: 4, - MinPeers: 5, - InitStreams: 5, - DiscSoftLowCap: 5, - DiscHardLowCap: 5, - DiscHighCap: 1024, - DiscBatch: 8, + Enabled: true, + Downloader: true, + StagedSync: false, + StagedSyncCfg: defaultStagedSyncConfig, + Concurrency: 4, + MinPeers: 5, + InitStreams: 5, + MaxAdvertiseWaitTime: 5, //minutes + DiscSoftLowCap: 5, + DiscHardLowCap: 5, + DiscHighCap: 1024, + DiscBatch: 8, + } + + defaultPartnerSyncConfig = harmonyconfig.SyncConfig{ + Enabled: true, + Downloader: true, + StagedSync: false, + StagedSyncCfg: defaultStagedSyncConfig, + Concurrency: 4, + MinPeers: 2, + InitStreams: 2, + MaxAdvertiseWaitTime: 2, //minutes + DiscSoftLowCap: 2, + DiscHardLowCap: 2, + DiscHighCap: 1024, + DiscBatch: 4, } defaultElseSyncConfig = harmonyconfig.SyncConfig{ - Enabled: true, - Downloader: true, - StagedSync: false, - StagedSyncCfg: defaultStagedSyncConfig, - Concurrency: 4, - MinPeers: 4, - InitStreams: 4, - DiscSoftLowCap: 4, - DiscHardLowCap: 4, - DiscHighCap: 1024, - DiscBatch: 8, + Enabled: true, + Downloader: true, + StagedSync: false, + StagedSyncCfg: defaultStagedSyncConfig, + Concurrency: 4, + MinPeers: 4, + InitStreams: 4, + MaxAdvertiseWaitTime: 2, //minutes + DiscSoftLowCap: 4, + DiscHardLowCap: 4, + DiscHighCap: 1024, + DiscBatch: 8, } ) diff --git a/cmd/harmony/flags.go b/cmd/harmony/flags.go index 26a250d951..168081660c 100644 --- a/cmd/harmony/flags.go +++ b/cmd/harmony/flags.go @@ -1692,6 +1692,11 @@ var ( Usage: "Initial shard-wise number of peers to start syncing", Hidden: true, } + syncMaxAdvertiseWaitTimeFlag = cli.IntFlag{ + Name: "sync.max-advertise-wait-time", + Usage: "The max time duration between two advertises for each p2p peer to tell other nodes what protocols it supports", + Hidden: true, + } syncDiscSoftLowFlag = cli.IntFlag{ Name: "sync.disc.soft-low-cap", Usage: "Soft low cap for sync stream management", @@ -1740,6 +1745,10 @@ func applySyncFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { config.Sync.InitStreams = cli.GetIntFlagValue(cmd, syncInitStreamsFlag) } + if cli.IsFlagChanged(cmd, syncMaxAdvertiseWaitTimeFlag) { + config.Sync.MaxAdvertiseWaitTime = cli.GetIntFlagValue(cmd, syncMaxAdvertiseWaitTimeFlag) + } + if cli.IsFlagChanged(cmd, syncDiscSoftLowFlag) { config.Sync.DiscSoftLowCap = cli.GetIntFlagValue(cmd, syncDiscSoftLowFlag) } diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 6ab3ed1024..dbb18227d2 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -918,16 +918,17 @@ func setupStagedSyncService(node *node.Node, host p2p.Host, hc harmonyconfig.Har } sConfig := stagedstreamsync.Config{ - ServerOnly: !hc.Sync.Downloader, - Network: nodeconfig.NetworkType(hc.Network.NetworkType), - Concurrency: hc.Sync.Concurrency, - MinStreams: hc.Sync.MinPeers, - InitStreams: hc.Sync.InitStreams, - SmSoftLowCap: hc.Sync.DiscSoftLowCap, - SmHardLowCap: hc.Sync.DiscHardLowCap, - SmHiCap: hc.Sync.DiscHighCap, - SmDiscBatch: hc.Sync.DiscBatch, - LogProgress: node.NodeConfig.LogProgress, + ServerOnly: !hc.Sync.Downloader, + Network: nodeconfig.NetworkType(hc.Network.NetworkType), + Concurrency: hc.Sync.Concurrency, + MinStreams: hc.Sync.MinPeers, + InitStreams: hc.Sync.InitStreams, + MaxAdvertiseWaitTime: hc.Sync.MaxAdvertiseWaitTime, + SmSoftLowCap: hc.Sync.DiscSoftLowCap, + SmHardLowCap: hc.Sync.DiscHardLowCap, + SmHiCap: hc.Sync.DiscHighCap, + SmDiscBatch: hc.Sync.DiscBatch, + LogProgress: node.NodeConfig.LogProgress, } // If we are running side chain, we will need to do some extra works for beacon diff --git a/internal/configs/harmony/harmony.go b/internal/configs/harmony/harmony.go index 90569b96f4..73ddd442dd 100644 --- a/internal/configs/harmony/harmony.go +++ b/internal/configs/harmony/harmony.go @@ -222,17 +222,18 @@ type PrometheusConfig struct { type SyncConfig struct { // TODO: Remove this bool after stream sync is fully up. - Enabled bool // enable the stream sync protocol - Downloader bool // start the sync downloader client - StagedSync bool // use staged sync - StagedSyncCfg StagedSyncConfig // staged sync configurations - Concurrency int // concurrency used for stream sync protocol - MinPeers int // minimum streams to start a sync task. - InitStreams int // minimum streams in bootstrap to start sync loop. - DiscSoftLowCap int // when number of streams is below this value, spin discover during check - DiscHardLowCap int // when removing stream, num is below this value, spin discovery immediately - DiscHighCap int // upper limit of streams in one sync protocol - DiscBatch int // size of each discovery + Enabled bool // enable the stream sync protocol + Downloader bool // start the sync downloader client + StagedSync bool // use staged sync + StagedSyncCfg StagedSyncConfig // staged sync configurations + Concurrency int // concurrency used for stream sync protocol + MinPeers int // minimum streams to start a sync task. + InitStreams int // minimum streams in bootstrap to start sync loop. + MaxAdvertiseWaitTime int // maximum time duration between advertisements + DiscSoftLowCap int // when number of streams is below this value, spin discover during check + DiscHardLowCap int // when removing stream, num is below this value, spin discovery immediately + DiscHighCap int // upper limit of streams in one sync protocol + DiscBatch int // size of each discovery } type StagedSyncConfig struct { diff --git a/p2p/stream/protocols/sync/protocol.go b/p2p/stream/protocols/sync/protocol.go index 2a984a5818..80ea0927bc 100644 --- a/p2p/stream/protocols/sync/protocol.go +++ b/p2p/stream/protocols/sync/protocol.go @@ -59,12 +59,13 @@ type ( // Config is the sync protocol config Config struct { - Chain engine.ChainReader - Host libp2p_host.Host - Discovery discovery.Discovery - ShardID nodeconfig.ShardID - Network nodeconfig.NetworkType - BeaconNode bool + Chain engine.ChainReader + Host libp2p_host.Host + Discovery discovery.Discovery + ShardID nodeconfig.ShardID + Network nodeconfig.NetworkType + BeaconNode bool + MaxAdvertiseWaitTime int // stream manager config SmSoftLowCap int SmHardLowCap int @@ -186,6 +187,10 @@ func (p *Protocol) HandleStream(raw libp2p_network.Stream) { func (p *Protocol) advertiseLoop() { for { sleep := p.advertise() + maxSleepTime := time.Duration(p.config.MaxAdvertiseWaitTime) * time.Minute + if sleep > maxSleepTime { + sleep = maxSleepTime + } select { case <-p.closeC: return diff --git a/rosetta/infra/harmony-mainnet.conf b/rosetta/infra/harmony-mainnet.conf index ddb2f87eea..3e7085ba63 100644 --- a/rosetta/infra/harmony-mainnet.conf +++ b/rosetta/infra/harmony-mainnet.conf @@ -1,4 +1,4 @@ -Version = "2.5.10" +Version = "2.5.11" [BLSKeys] KMSConfigFile = "" @@ -105,6 +105,7 @@ Version = "2.5.10" Enabled = false InitStreams = 8 MinPeers = 5 + MaxAdvertiseWaitTime = 30 [TxPool] AccountSlots = 16 diff --git a/rosetta/infra/harmony-pstn.conf b/rosetta/infra/harmony-pstn.conf index 1c029f8281..458e826354 100644 --- a/rosetta/infra/harmony-pstn.conf +++ b/rosetta/infra/harmony-pstn.conf @@ -1,4 +1,4 @@ -Version = "2.5.10" +Version = "2.5.11" [BLSKeys] KMSConfigFile = "" @@ -105,6 +105,7 @@ Version = "2.5.10" Enabled = false InitStreams = 8 MinPeers = 2 + MaxAdvertiseWaitTime = 30 [TxPool] AccountSlots = 16 From d23d98f6898365c3d6d5bf9d5758b56771b1a35b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Mon, 16 Jan 2023 23:18:02 +0800 Subject: [PATCH 043/420] terminal print the peer id and proto id --- p2p/stream/common/streammanager/streammanager.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/p2p/stream/common/streammanager/streammanager.go b/p2p/stream/common/streammanager/streammanager.go index 98e20cbaf9..e57919a76f 100644 --- a/p2p/stream/common/streammanager/streammanager.go +++ b/p2p/stream/common/streammanager/streammanager.go @@ -73,8 +73,12 @@ func newStreamManager(pid sttypes.ProtoID, host host, pf peerFinder, handleStrea Str("protocol ID", string(pid)).Logger() protoSpec, _ := sttypes.ProtoIDToProtoSpec(pid) - fmt.Println("my peer id: ", host.ID().String()) - fmt.Println("my proto id: ", pid) + + // if it is a beacon node or shard node, print the peer id and proto id + if protoSpec.BeaconNode || protoSpec.ShardID != shard.BeaconChainShardID { + fmt.Println("My peer id: ", host.ID().String()) + fmt.Println("My proto id: ", pid) + } return &streamManager{ myProtoID: pid, From f1d49a5c87a03d819406378895f26e4bece04e1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Thu, 19 Jan 2023 23:42:29 +0800 Subject: [PATCH 044/420] fix boot complete message when node is shut down --- api/service/stagedstreamsync/downloader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go index cfcd180c0f..a20b4ac79a 100644 --- a/api/service/stagedstreamsync/downloader.go +++ b/api/service/stagedstreamsync/downloader.go @@ -95,7 +95,6 @@ func NewDownloader(host p2p.Host, bc core.BlockChain, isBeaconNode bool, config func (d *Downloader) Start() { go func() { d.waitForBootFinish() - fmt.Printf("boot completed for shard %d, %d streams are connected\n", d.bc.ShardID(), d.syncProtocol.NumStreams()) d.loop() }() @@ -178,6 +177,7 @@ func (d *Downloader) waitForBootFinish() { case <-checkCh: if d.syncProtocol.NumStreams() >= d.config.InitStreams { + fmt.Printf("boot completed for shard %d ( %d streams are connected )\n", d.bc.ShardID(), d.syncProtocol.NumStreams()) return } case <-d.closeC: From 41d72b4c4ee93dc29bfc259adf9cc39ecea65875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Fri, 20 Jan 2023 10:50:03 +0800 Subject: [PATCH 045/420] add new config option ( ForceReachabilityPublic ) to fix local-net consensus issue --- cmd/bootnode/main.go | 13 ++++++---- cmd/harmony/default.go | 10 ++++---- cmd/harmony/main.go | 8 ++++++ p2p/host.go | 57 ++++++++++++++++++++++++++++++------------ test/deploy.sh | 2 +- 5 files changed, 63 insertions(+), 27 deletions(-) diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index 6e3a8bbbc5..92d6674b94 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -102,6 +102,7 @@ func main() { verbosity := flag.Int("verbosity", 5, "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail (default: 5)") logConn := flag.Bool("log_conn", false, "log incoming/outgoing connections") maxConnPerIP := flag.Int("max_conn_per_ip", 10, "max connections number for same ip") + forceReachabilityPublic := flag.Bool("force_public", false, "forcing the local node to believe it is reachable externally") flag.Parse() @@ -124,12 +125,14 @@ func main() { // For bootstrap nodes, we shall keep .dht file. dataStorePath := fmt.Sprintf(".dht-%s-%s", *ip, *port) selfPeer := p2p.Peer{IP: *ip, Port: *port} + host, err := p2p.NewHost(p2p.HostConfig{ - Self: &selfPeer, - BLSKey: privKey, - BootNodes: nil, // Boot nodes have no boot nodes :) Will be connected when other nodes joined - DataStoreFile: &dataStorePath, - MaxConnPerIP: *maxConnPerIP, + Self: &selfPeer, + BLSKey: privKey, + BootNodes: nil, // Boot nodes have no boot nodes :) Will be connected when other nodes joined + DataStoreFile: &dataStorePath, + MaxConnPerIP: *maxConnPerIP, + ForceReachabilityPublic: *forceReachabilityPublic, }) if err != nil { utils.FatalErrMsg(err, "cannot initialize network") diff --git a/cmd/harmony/default.go b/cmd/harmony/default.go index 7bdcc02d0d..2a6c7a75c3 100644 --- a/cmd/harmony/default.go +++ b/cmd/harmony/default.go @@ -191,14 +191,14 @@ var ( defaultLocalNetSyncConfig = harmonyconfig.SyncConfig{ Enabled: true, Downloader: true, - StagedSync: false, + StagedSync: true, StagedSyncCfg: defaultStagedSyncConfig, Concurrency: 4, - MinPeers: 5, - InitStreams: 5, + MinPeers: 4, + InitStreams: 4, MaxAdvertiseWaitTime: 5, //minutes - DiscSoftLowCap: 5, - DiscHardLowCap: 5, + DiscSoftLowCap: 4, + DiscHardLowCap: 4, DiscHighCap: 1024, DiscBatch: 8, } diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index dbb18227d2..8543158b3a 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -629,6 +629,13 @@ func createGlobalConfig(hc harmonyconfig.HarmonyConfig) (*nodeconfig.ConfigType, Port: strconv.Itoa(hc.P2P.Port), ConsensusPubKey: nodeConfig.ConsensusPriKey[0].Pub.Object, } + + // for local-net the node has to be forced to assume it is public reachable + forceReachabilityPublic := false + if hc.Network.NetworkType == nodeconfig.Localnet { + forceReachabilityPublic = true + } + myHost, err = p2p.NewHost(p2p.HostConfig{ Self: &selfPeer, BLSKey: nodeConfig.P2PPriKey, @@ -639,6 +646,7 @@ func createGlobalConfig(hc harmonyconfig.HarmonyConfig) (*nodeconfig.ConfigType, DisablePrivateIPScan: hc.P2P.DisablePrivateIPScan, MaxPeers: hc.P2P.MaxPeers, WaitForEachPeerToConnect: hc.P2P.WaitForEachPeerToConnect, + ForceReachabilityPublic: forceReachabilityPublic, }) if err != nil { return nil, errors.Wrap(err, "cannot create P2P network host") diff --git a/p2p/host.go b/p2p/host.go index fcd2ea3b4d..bab1da691c 100644 --- a/p2p/host.go +++ b/p2p/host.go @@ -16,6 +16,7 @@ import ( "github.com/libp2p/go-libp2p-core/routing" dht "github.com/libp2p/go-libp2p-kad-dht" libp2p_pubsub "github.com/libp2p/go-libp2p-pubsub" + libp2p_config "github.com/libp2p/go-libp2p/config" libp2p_crypto "github.com/libp2p/go-libp2p/core/crypto" libp2p_host "github.com/libp2p/go-libp2p/core/host" libp2p_network "github.com/libp2p/go-libp2p/core/network" @@ -96,6 +97,7 @@ type HostConfig struct { DisablePrivateIPScan bool MaxPeers int64 WaitForEachPeerToConnect bool + ForceReachabilityPublic bool } func init() { @@ -112,6 +114,19 @@ func init() { libp2p_pubsub.GossipSubMaxIHaveLength = 1000 } +func forceReachabilityPublic(f bool) libp2p_config.Option { + if f { + return func(cfg *libp2p_config.Config) error { + public := libp2p_network.Reachability(libp2p_network.ReachabilityPublic) + cfg.AutoNATConfig.ForceReachability = &public + return nil + } + } + return func(p2pConfig *libp2p_config.Config) error { + return nil + } +} + // NewHost .. func NewHost(cfg HostConfig) (Host, error) { var ( @@ -128,8 +143,8 @@ func NewHost(cfg HostConfig) (Host, error) { ctx, cancel := context.WithCancel(context.Background()) // TODO: move low and high to configs connmgr, err := connmgr.NewConnManager( - int(10), // LowWater - int(10000)*cfg.MaxConnPerIP, // HighWater, + int(cfg.MaxConnPerIP), // LowWater + int(1024)*cfg.MaxConnPerIP, // HighWater, connmgr.WithGracePeriod(time.Minute), ) if err != nil { @@ -138,16 +153,16 @@ func NewHost(cfg HostConfig) (Host, error) { } var idht *dht.IpfsDHT var opt discovery.DHTConfig - p2pHost, err := libp2p.New( + p2pHostConfig := []libp2p.Option{ listenAddr, libp2p.Identity(key), - // support TLS connections + // Support TLS connections libp2p.Security(libp2ptls.ID, libp2ptls.New), - // support noise connections + // Support noise connections libp2p.Security(noise.ID, noise.New), - // support any other default transports (TCP) + // Support any other default transports (TCP) libp2p.DefaultTransports, - // Let's prevent our peer from having too many + // Prevent the peer from having too many // connections by attaching a connection manager. libp2p.ConnectionManager(connmgr), // Attempt to open ports using uPNP for NATed hosts. @@ -165,8 +180,7 @@ func NewHost(cfg HostConfig) (Host, error) { idht, err = dht.New(ctx, h, opts...) return idht, err }), - - // to help other peers to figure out if they are behind + // To help other peers to figure out if they are behind // NATs, launch the server-side of AutoNAT too (AutoRelay // already runs the client) // This service is highly rate-limited and should not cause any @@ -174,14 +188,23 @@ func NewHost(cfg HostConfig) (Host, error) { libp2p.EnableNATService(), // Bandwidth Reporter libp2p.BandwidthReporter(newCounter()), - // ForceReachabilityPublic overrides automatic reachability detection in the AutoNAT subsystem, - // forcing the local node to believe it is reachable externally. - // libp2p.ForceReachabilityPublic(), - // libp2p.DisableRelay(), + // Enable relay service, to disable relay we can use libp2p.DisableRelay() libp2p.EnableRelayService(), - // prevent dialing of public addresses - // libp2p.ConnectionGater(NewGater(cfg.DisablePrivateIPScan)), - ) + } + + if cfg.ForceReachabilityPublic { + // ForceReachabilityPublic overrides automatic reachability detection in the AutoNAT subsystem, + // forcing the local node to believe it is reachable externally + p2pHostConfig = append(p2pHostConfig, libp2p.ForceReachabilityPublic()) + } + + if cfg.DisablePrivateIPScan { + // Prevent dialing of public addresses + p2pHostConfig = append(p2pHostConfig, libp2p.ConnectionGater(NewGater(cfg.DisablePrivateIPScan))) + } + + // create p2p host + p2pHost, err := libp2p.New(p2pHostConfig...) if err != nil { cancel() return nil, errors.Wrapf(err, "cannot initialize libp2p host") @@ -190,6 +213,7 @@ func NewHost(cfg HostConfig) (Host, error) { disc, err := discovery.NewDHTDiscovery(ctx, cancel, p2pHost, idht, opt) if err != nil { cancel() + p2pHost.Close() return nil, errors.Wrap(err, "cannot create DHT discovery") } @@ -230,6 +254,7 @@ func NewHost(cfg HostConfig) (Host, error) { pubsub, err := libp2p_pubsub.NewGossipSub(ctx, p2pHost, options...) if err != nil { cancel() + p2pHost.Close() return nil, errors.Wrapf(err, "cannot initialize libp2p pub-sub") } diff --git a/test/deploy.sh b/test/deploy.sh index 6bbb12eb9b..9ab6943110 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -54,7 +54,7 @@ function setup() { function launch_bootnode() { echo "launching boot node ..." - ${DRYRUN} ${ROOT}/bin/bootnode -port 19876 -max_conn_per_ip 100 >"${log_folder}"/bootnode.log 2>&1 | tee -a "${LOG_FILE}" & + ${DRYRUN} ${ROOT}/bin/bootnode -port 19876 -max_conn_per_ip 100 -force_public true >"${log_folder}"/bootnode.log 2>&1 | tee -a "${LOG_FILE}" & sleep 1 BN_MA=$(grep "BN_MA" "${log_folder}"/bootnode.log | awk -F\= ' { print $2 } ') echo "bootnode launched." + " $BN_MA" From 94e3c52bd7bc165bc8f98d4c857af9c985821188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Thu, 26 Jan 2023 11:17:45 +0800 Subject: [PATCH 046/420] fix self query issue --- api/service/legacysync/epoch_syncing.go | 6 +++-- api/service/legacysync/helpers.go | 6 +++-- api/service/legacysync/syncing.go | 24 ++++++++++++-------- api/service/legacysync/syncing_test.go | 4 +++- api/service/stagedsync/stagedsync.go | 10 +++++++-- api/service/stagedsync/sync_config.go | 7 ++++++ cmd/harmony/main.go | 3 ++- node/node.go | 2 +- node/node_syncing.go | 29 +++++++++++++++++++++---- 9 files changed, 69 insertions(+), 22 deletions(-) diff --git a/api/service/legacysync/epoch_syncing.go b/api/service/legacysync/epoch_syncing.go index 8bf7ae1458..641fb58890 100644 --- a/api/service/legacysync/epoch_syncing.go +++ b/api/service/legacysync/epoch_syncing.go @@ -14,6 +14,8 @@ import ( "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/shard" "github.com/pkg/errors" + + libp2p_peer "github.com/libp2p/go-libp2p/core/peer" ) type EpochSync struct { @@ -202,8 +204,8 @@ func processWithPayload(payload [][]byte, bc core.BlockChain) error { } // CreateSyncConfig creates SyncConfig for StateSync object. -func (ss *EpochSync) CreateSyncConfig(peers []p2p.Peer, shardID uint32, waitForEachPeerToConnect bool) error { +func (ss *EpochSync) CreateSyncConfig(peers []p2p.Peer, shardID uint32, selfPeerID libp2p_peer.ID, waitForEachPeerToConnect bool) error { var err error - ss.syncConfig, err = createSyncConfig(ss.syncConfig, peers, shardID, waitForEachPeerToConnect) + ss.syncConfig, err = createSyncConfig(ss.syncConfig, peers, shardID, selfPeerID, waitForEachPeerToConnect) return err } diff --git a/api/service/legacysync/helpers.go b/api/service/legacysync/helpers.go index ca66e0ddcb..4699257f8b 100644 --- a/api/service/legacysync/helpers.go +++ b/api/service/legacysync/helpers.go @@ -9,6 +9,7 @@ import ( "github.com/harmony-one/harmony/api/service/legacysync/downloader" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" + libp2p_peer "github.com/libp2p/go-libp2p/core/peer" "github.com/pkg/errors" ) @@ -51,7 +52,7 @@ func getMaxPeerHeight(syncConfig *SyncConfig) uint64 { return maxHeight } -func createSyncConfig(syncConfig *SyncConfig, peers []p2p.Peer, shardID uint32, waitForEachPeerToConnect bool) (*SyncConfig, error) { +func createSyncConfig(syncConfig *SyncConfig, peers []p2p.Peer, shardID uint32, selfPeerID libp2p_peer.ID, waitForEachPeerToConnect bool) (*SyncConfig, error) { // sanity check to ensure no duplicate peers if err := checkPeersDuplicity(peers); err != nil { return syncConfig, err @@ -61,6 +62,7 @@ func createSyncConfig(syncConfig *SyncConfig, peers []p2p.Peer, shardID uint32, targetSize, peers := limitNumPeers(peers, randSeed) utils.Logger().Debug(). + Str("self peer ID", string(selfPeerID)). Int("peers count", len(peers)). Int("target size", targetSize). Uint32("shardID", shardID). @@ -72,7 +74,7 @@ func createSyncConfig(syncConfig *SyncConfig, peers []p2p.Peer, shardID uint32, if syncConfig != nil { syncConfig.CloseConnections() } - syncConfig = NewSyncConfig(shardID, nil) + syncConfig = NewSyncConfig(shardID, selfPeerID, nil) if !waitForEachPeerToConnect { var wg sync.WaitGroup diff --git a/api/service/legacysync/syncing.go b/api/service/legacysync/syncing.go index 8afaa3a95b..0f004bfb4f 100644 --- a/api/service/legacysync/syncing.go +++ b/api/service/legacysync/syncing.go @@ -25,6 +25,7 @@ import ( "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/node/worker" "github.com/harmony-one/harmony/p2p" + libp2p_peer "github.com/libp2p/go-libp2p/core/peer" "github.com/pkg/errors" ) @@ -110,14 +111,16 @@ type SyncConfig struct { // SyncPeerConfig itself is guarded by its own mutex. mtx sync.RWMutex - peers []*SyncPeerConfig - shardID uint32 + peers []*SyncPeerConfig + shardID uint32 + selfPeerID libp2p_peer.ID } -func NewSyncConfig(shardID uint32, peers []*SyncPeerConfig) *SyncConfig { +func NewSyncConfig(shardID uint32, selfPeerID libp2p_peer.ID, peers []*SyncPeerConfig) *SyncConfig { return &SyncConfig{ - peers: peers, - shardID: shardID, + peers: peers, + shardID: shardID, + selfPeerID: selfPeerID, } } @@ -135,6 +138,9 @@ func (sc *SyncConfig) AddPeer(peer *SyncPeerConfig) { if peer.IsEqual(p2) { return } + if peer.peer.PeerID == sc.selfPeerID { + return + } } sc.peers = append(sc.peers, peer) } @@ -192,7 +198,7 @@ func (sc *SyncConfig) RemovePeer(peer *SyncPeerConfig, reason string) { } // CreateStateSync returns the implementation of StateSyncInterface interface. -func CreateStateSync(bc blockChain, ip string, port string, peerHash [20]byte, isExplorer bool, role nodeconfig.Role) *StateSync { +func CreateStateSync(bc blockChain, ip string, port string, peerHash [20]byte, peerID libp2p_peer.ID, isExplorer bool, role nodeconfig.Role) *StateSync { stateSync := &StateSync{} stateSync.blockChain = bc stateSync.selfip = ip @@ -201,7 +207,7 @@ func CreateStateSync(bc blockChain, ip string, port string, peerHash [20]byte, i stateSync.commonBlocks = make(map[int]*types.Block) stateSync.lastMileBlocks = []*types.Block{} stateSync.isExplorer = isExplorer - stateSync.syncConfig = NewSyncConfig(bc.ShardID(), nil) + stateSync.syncConfig = NewSyncConfig(bc.ShardID(), peerID, nil) stateSync.syncStatus = newSyncStatus(role) return stateSync @@ -366,9 +372,9 @@ func (peerConfig *SyncPeerConfig) GetBlocks(hashes [][]byte) ([][]byte, error) { } // CreateSyncConfig creates SyncConfig for StateSync object. -func (ss *StateSync) CreateSyncConfig(peers []p2p.Peer, shardID uint32, waitForEachPeerToConnect bool) error { +func (ss *StateSync) CreateSyncConfig(peers []p2p.Peer, shardID uint32, selfPeerID libp2p_peer.ID, waitForEachPeerToConnect bool) error { var err error - ss.syncConfig, err = createSyncConfig(ss.syncConfig, peers, shardID, waitForEachPeerToConnect) + ss.syncConfig, err = createSyncConfig(ss.syncConfig, peers, shardID, selfPeerID, waitForEachPeerToConnect) return err } diff --git a/api/service/legacysync/syncing_test.go b/api/service/legacysync/syncing_test.go index bcab494a18..bc17aeaec6 100644 --- a/api/service/legacysync/syncing_test.go +++ b/api/service/legacysync/syncing_test.go @@ -12,6 +12,7 @@ import ( "time" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + peer "github.com/libp2p/go-libp2p-core/peer" "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/harmony/api/service/legacysync/downloader" @@ -128,7 +129,8 @@ func (mockBlockchain) ShardID() uint32 { } func TestCreateStateSync(t *testing.T) { - stateSync := CreateStateSync(mockBlockchain{}, "127.0.0.1", "8000", [20]byte{}, false, nodeconfig.Validator) + pID, _ := peer.IDFromBytes([]byte{}) + stateSync := CreateStateSync(mockBlockchain{}, "127.0.0.1", "8000", [20]byte{}, pID, false, nodeconfig.Validator) if stateSync == nil { t.Error("Unable to create stateSync") diff --git a/api/service/stagedsync/stagedsync.go b/api/service/stagedsync/stagedsync.go index 88df1a671f..c81f0b5b12 100644 --- a/api/service/stagedsync/stagedsync.go +++ b/api/service/stagedsync/stagedsync.go @@ -26,6 +26,8 @@ import ( "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" "github.com/ledgerwatch/erigon-lib/kv" + + libp2p_peer "github.com/libp2p/go-libp2p/core/peer" ) type StagedSync struct { @@ -663,7 +665,7 @@ func (ss *StagedSync) AddNewBlock(peerHash []byte, block *types.Block) { } // CreateSyncConfig creates SyncConfig for StateSync object. -func (ss *StagedSync) CreateSyncConfig(peers []p2p.Peer, shardID uint32, waitForEachPeerToConnect bool) error { +func (ss *StagedSync) CreateSyncConfig(peers []p2p.Peer, shardID uint32, selfPeerID libp2p_peer.ID, waitForEachPeerToConnect bool) error { // sanity check to ensure no duplicate peers if err := checkPeersDuplicity(peers); err != nil { return err @@ -678,6 +680,7 @@ func (ss *StagedSync) CreateSyncConfig(peers []p2p.Peer, shardID uint32, waitFor } utils.Logger().Debug(). + Str("self peer ID", string(selfPeerID)). Int("peers count", len(peers)). Int("target size", targetSize). Msg("[STAGED_SYNC] CreateSyncConfig: len of peers") @@ -685,7 +688,9 @@ func (ss *StagedSync) CreateSyncConfig(peers []p2p.Peer, shardID uint32, waitFor if ss.syncConfig != nil { ss.syncConfig.CloseConnections() } - ss.syncConfig = &SyncConfig{} + ss.syncConfig = &SyncConfig{ + selfPeerID: selfPeerID, + } var connectedPeers int for _, peer := range peers { @@ -694,6 +699,7 @@ func (ss *StagedSync) CreateSyncConfig(peers []p2p.Peer, shardID uint32, waitFor continue } peerConfig := &SyncPeerConfig{ + peer: peer, ip: peer.IP, port: peer.Port, client: client, diff --git a/api/service/stagedsync/sync_config.go b/api/service/stagedsync/sync_config.go index f42737cc1a..91b3a4d739 100644 --- a/api/service/stagedsync/sync_config.go +++ b/api/service/stagedsync/sync_config.go @@ -14,6 +14,8 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" + + libp2p_peer "github.com/libp2p/go-libp2p/core/peer" ) // Constants for syncing. @@ -40,6 +42,7 @@ const ( // SyncPeerConfig is peer config to sync. type SyncPeerConfig struct { + peer p2p.Peer ip string port string peerHash []byte @@ -156,6 +159,7 @@ type SyncConfig struct { mtx sync.RWMutex reservedPeers []*SyncPeerConfig peers []*SyncPeerConfig + selfPeerID libp2p_peer.ID } // AddPeer adds the given sync peer. @@ -168,6 +172,9 @@ func (sc *SyncConfig) AddPeer(peer *SyncPeerConfig) { if peer.IsEqual(p2) { return } + if peer.peer.PeerID == sc.selfPeerID { + return + } } sc.peers = append(sc.peers, peer) } diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 8543158b3a..b68e757422 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -773,7 +773,8 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi currentNode.SyncingPeerProvider = node.NewLocalSyncingPeerProvider( 6000, uint16(selfPort), epochConfig.NumShards(), uint32(epochConfig.NumNodesPerShard())) } else { - currentNode.SyncingPeerProvider = node.NewDNSSyncingPeerProvider(hc.DNSSync.Zone, strconv.Itoa(hc.DNSSync.Port)) + addrs := myHost.GetP2PHost().Addrs() + currentNode.SyncingPeerProvider = node.NewDNSSyncingPeerProvider(hc.DNSSync.Zone, strconv.Itoa(hc.DNSSync.Port), addrs) } currentNode.NodeConfig.DNSZone = hc.DNSSync.Zone diff --git a/node/node.go b/node/node.go index bc80ef81ae..6c2ebcd5fe 100644 --- a/node/node.go +++ b/node/node.go @@ -87,7 +87,7 @@ type ISync interface { UpdateBlockAndStatus(block *types.Block, bc core.BlockChain, verifyAllSig bool) error AddLastMileBlock(block *types.Block) GetActivePeerNumber() int - CreateSyncConfig(peers []p2p.Peer, shardID uint32, waitForEachPeerToConnect bool) error + CreateSyncConfig(peers []p2p.Peer, shardID uint32, selfPeerID libp2p_peer.ID, waitForEachPeerToConnect bool) error SyncLoop(bc core.BlockChain, worker *worker.Worker, isBeacon bool, consensus *consensus.Consensus, loopMinTime time.Duration) IsSynchronized() bool IsSameBlockchainHeight(bc core.BlockChain) (uint64, bool) diff --git a/node/node_syncing.go b/node/node_syncing.go index 2219be96da..a0a8e6c84c 100644 --- a/node/node_syncing.go +++ b/node/node_syncing.go @@ -9,6 +9,7 @@ import ( "time" "github.com/harmony-one/harmony/internal/tikv" + "github.com/multiformats/go-multiaddr" prom "github.com/harmony-one/harmony/api/service/prometheus" "github.com/prometheus/client_golang/prometheus" @@ -105,7 +106,8 @@ func (node *Node) createStateSync(bc core.BlockChain) *legacysync.StateSync { mutatedPort := strconv.Itoa(mySyncPort + legacysync.SyncingPortDifference) role := node.NodeConfig.Role() return legacysync.CreateStateSync(bc, node.SelfPeer.IP, mutatedPort, - node.GetSyncID(), node.NodeConfig.Role() == nodeconfig.ExplorerNode, role) + node.GetSyncID(), node.host.GetID(), + node.NodeConfig.Role() == nodeconfig.ExplorerNode, role) } func (node *Node) createStagedSync(bc core.BlockChain) *stagedsync.StagedSync { @@ -151,14 +153,16 @@ type SyncingPeerProvider interface { // DNSSyncingPeerProvider uses the given DNS zone to resolve syncing peers. type DNSSyncingPeerProvider struct { + selfAddrs []multiaddr.Multiaddr zone, port string lookupHost func(name string) (addrs []string, err error) } // NewDNSSyncingPeerProvider returns a provider that uses given DNS name and // port number to resolve syncing peers. -func NewDNSSyncingPeerProvider(zone, port string) *DNSSyncingPeerProvider { +func NewDNSSyncingPeerProvider(zone, port string, addrs []multiaddr.Multiaddr) *DNSSyncingPeerProvider { return &DNSSyncingPeerProvider{ + selfAddrs: addrs, zone: zone, port: port, lookupHost: net.LookupHost, @@ -174,11 +178,27 @@ func (p *DNSSyncingPeerProvider) SyncingPeers(shardID uint32) (peers []p2p.Peer, "[SYNC] cannot find peers using DNS name %#v", dns) } for _, addr := range addrs { + // no need to have peer itself on the list of connected peers + if p.getSelfAddrIndex(addr, p.port) >= 0 { + continue + } peers = append(peers, p2p.Peer{IP: addr, Port: p.port}) } return peers, nil } +// getSelfAddrIndex returns address index if it is one of self addresses +func (p *DNSSyncingPeerProvider) getSelfAddrIndex(IP string, Port string) int { + peerAddr4 := fmt.Sprintf("/ip4/%s/tcp/%s", IP, Port) + peerAddr6 := fmt.Sprintf("/ip6/%s/tcp/%s", IP, Port) + for addrIndex, addr := range p.selfAddrs { + if addr.String() == peerAddr4 || addr.String() == peerAddr6 { + return addrIndex + } + } + return -1 +} + // LocalSyncingPeerProvider uses localnet deployment convention to synthesize // syncing peers. type LocalSyncingPeerProvider struct { @@ -253,7 +273,8 @@ func (node *Node) doBeaconSyncing() { continue } - if err := node.epochSync.CreateSyncConfig(peers, shard.BeaconChainShardID, node.HarmonyConfig.P2P.WaitForEachPeerToConnect); err != nil { + if err := node.epochSync.CreateSyncConfig(peers, shard.BeaconChainShardID, node.host.GetID(), + node.HarmonyConfig.P2P.WaitForEachPeerToConnect); err != nil { utils.Logger().Warn().Err(err).Msg("[EPOCHSYNC] cannot create beacon sync config") continue } @@ -296,7 +317,7 @@ func (node *Node) doSync(bc core.BlockChain, worker *worker.Worker, willJoinCons Msg("cannot retrieve syncing peers") return } - if err := syncInstance.CreateSyncConfig(peers, shardID, node.HarmonyConfig.P2P.WaitForEachPeerToConnect); err != nil { + if err := syncInstance.CreateSyncConfig(peers, shardID, node.host.GetID(), node.HarmonyConfig.P2P.WaitForEachPeerToConnect); err != nil { utils.Logger().Warn(). Err(err). Interface("peers", peers). From b94e2cf7078e1467c19bdcee1afa2b20a55f33f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Thu, 26 Jan 2023 13:53:47 +0800 Subject: [PATCH 047/420] fix test NewDNSSyncingPeerProvider --- node/node_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/node/node_test.go b/node/node_test.go index a4f1af70c3..49ba5d164d 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -16,6 +16,7 @@ import ( "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/shard" + "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/assert" ) @@ -69,7 +70,8 @@ func TestNewNode(t *testing.T) { func TestDNSSyncingPeerProvider(t *testing.T) { t.Run("Happy", func(t *testing.T) { - p := NewDNSSyncingPeerProvider("example.com", "1234") + addrs := make([]multiaddr.Multiaddr, 0) + p := NewDNSSyncingPeerProvider("example.com", "1234", addrs) lookupCount := 0 lookupName := "" p.lookupHost = func(name string) (addrs []string, err error) { @@ -92,7 +94,8 @@ func TestDNSSyncingPeerProvider(t *testing.T) { } }) t.Run("LookupError", func(t *testing.T) { - p := NewDNSSyncingPeerProvider("example.com", "1234") + addrs := make([]multiaddr.Multiaddr, 0) + p := NewDNSSyncingPeerProvider("example.com", "1234", addrs) p.lookupHost = func(_ string) ([]string, error) { return nil, errors.New("omg") } From 5fe1bac94d9a0a1b7e46fdd375b47cc505e15e2a Mon Sep 17 00:00:00 2001 From: "Nita Neou (Soph)" Date: Fri, 27 Jan 2023 16:20:50 +0700 Subject: [PATCH 048/420] Dockerfile update apk add software version --- scripts/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/docker/Dockerfile b/scripts/docker/Dockerfile index 6e13cd4992..5e85c98428 100644 --- a/scripts/docker/Dockerfile +++ b/scripts/docker/Dockerfile @@ -39,7 +39,7 @@ ARG HARMONY_USER_GID=1000 ENV HARMONY_HOME=/harmony ENV HOME=${HARMONY_HOME} -RUN apk add --no-cache bash~=5.1.16-r2 bind-tools~=9.16.29-r0 tini~=0.19.0 curl==7.83.1-r2 sed~=4.8-r0 \ +RUN apk add --no-cache bash~=5.1.16-r2 bind-tools~=9.16.37-r0 tini~=0.19.0 curl==7.83.1-r5 sed~=4.8-r0 \ && rm -rf /var/cache/apk/* \ && addgroup -g ${HARMONY_USER_GID} ${HARMONY_USER} \ && adduser -u ${HARMONY_USER_UID} -G ${HARMONY_USER} --shell /sbin/nologin --no-create-home -D ${HARMONY_USER} \ From a5db2076172559855ab924f2c38d69a0d3104a7c Mon Sep 17 00:00:00 2001 From: "Nita Neou (Soph)" Date: Fri, 27 Jan 2023 16:29:17 +0700 Subject: [PATCH 049/420] github action - update --- .github/workflows/ci-tag.yaml | 10 +++++----- .github/workflows/ci.yaml | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci-tag.yaml b/.github/workflows/ci-tag.yaml index 75ae96f532..d34e1b775e 100644 --- a/.github/workflows/ci-tag.yaml +++ b/.github/workflows/ci-tag.yaml @@ -146,7 +146,7 @@ jobs: working-directory: harmony - name: Download artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: harmony @@ -156,16 +156,16 @@ jobs: working-directory: harmony - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: context: ./harmony/scripts/docker file: ./harmony/scripts/docker/Dockerfile @@ -205,7 +205,7 @@ jobs: working-directory: harmony - name: Download artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: harmony diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1ebe5e4491..58e4743511 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -148,7 +148,7 @@ jobs: working-directory: harmony - name: Download artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: harmony @@ -158,16 +158,16 @@ jobs: working-directory: harmony - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: context: ./harmony/scripts/docker file: ./harmony/scripts/docker/Dockerfile @@ -207,7 +207,7 @@ jobs: working-directory: harmony - name: Download artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: harmony From bd6d7ca7db14257fb4be291582590e068041c46e Mon Sep 17 00:00:00 2001 From: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Date: Thu, 2 Feb 2023 13:11:40 +0000 Subject: [PATCH 050/420] [testnet] disable leader rotation --- internal/params/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/params/config.go b/internal/params/config.go index 3d40a7c6c9..c11daeafb5 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -109,7 +109,7 @@ var ( ChainIdFixEpoch: big.NewInt(0), CrossShardXferPrecompileEpoch: big.NewInt(2), AllowlistEpoch: big.NewInt(2), - LeaderRotationEpoch: big.NewInt(295), + LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 64, FeeCollectEpoch: EpochTBD, } From f07dd5681a2033092821421afcd0a267eefc40af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Fri, 10 Feb 2023 22:36:54 +0800 Subject: [PATCH 051/420] fix discovery issue for legacy sync --- api/service/legacysync/epoch_syncing.go | 2 +- p2p/discovery/option.go | 4 +++- p2p/stream/common/streammanager/cooldown.go | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/api/service/legacysync/epoch_syncing.go b/api/service/legacysync/epoch_syncing.go index 641fb58890..8d1cb37c4f 100644 --- a/api/service/legacysync/epoch_syncing.go +++ b/api/service/legacysync/epoch_syncing.go @@ -112,7 +112,7 @@ func syncLoop(bc core.BlockChain, syncConfig *SyncConfig) (timeout int) { } if otherEpoch < curEpoch { for _, peerCfg := range syncConfig.GetPeers() { - syncConfig.RemovePeer(peerCfg, fmt.Sprintf("[EPOCHSYNC]: current height is higher that others, remove peers: %s", peerCfg.String())) + syncConfig.RemovePeer(peerCfg, fmt.Sprintf("[EPOCHSYNC]: current height is higher than others, remove peers: %s", peerCfg.String())) } return 2 } diff --git a/p2p/discovery/option.go b/p2p/discovery/option.go index ce42592703..fff8eea20f 100644 --- a/p2p/discovery/option.go +++ b/p2p/discovery/option.go @@ -42,7 +42,9 @@ func (opt DHTConfig) GetLibp2pRawOptions() ([]libp2p_dht.Option, error) { opts = append(opts, libp2p_dht.Concurrency(opt.DiscConcurrency)) } - opts = append(opts, libp2p_dht.DisableAutoRefresh()) + // TODO: to disable auto refresh to make sure there is no conflicts with protocol discovery functions + // it's not applicable for legacy sync + // opts = append(opts, libp2p_dht.DisableAutoRefresh()) return opts, nil } diff --git a/p2p/stream/common/streammanager/cooldown.go b/p2p/stream/common/streammanager/cooldown.go index f22f4956f0..ef6d21bc98 100644 --- a/p2p/stream/common/streammanager/cooldown.go +++ b/p2p/stream/common/streammanager/cooldown.go @@ -36,6 +36,8 @@ func (cache *coolDownCache) Has(id peer.ID) bool { // Add adds the peer ID to the cache func (cache *coolDownCache) Add(id peer.ID) { + cache.mu.Lock() + defer cache.mu.Unlock() has := cache.timeCache.Has(string(id)) if !has { cache.timeCache.Add(string(id)) From 4b8bfabf284134a12947265a3d23310f2960e626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Mon, 13 Feb 2023 15:24:35 +0800 Subject: [PATCH 052/420] add watermark low/high options for p2p connection manager --- cmd/harmony/config_migrations.go | 6 +++ cmd/harmony/default.go | 2 + cmd/harmony/flags.go | 20 +++++++++ cmd/harmony/flags_test.go | 12 ++++++ cmd/harmony/main.go | 2 + internal/configs/harmony/harmony.go | 2 + internal/configs/node/network.go | 8 +++- node/node_handler.go | 2 + p2p/discovery/option_test.go | 4 +- p2p/host.go | 64 +++++++++++++++++++---------- rosetta/infra/harmony-mainnet.conf | 2 + rosetta/infra/harmony-pstn.conf | 2 + 12 files changed, 101 insertions(+), 25 deletions(-) diff --git a/cmd/harmony/config_migrations.go b/cmd/harmony/config_migrations.go index 3a256f49d2..bdbfefdc91 100644 --- a/cmd/harmony/config_migrations.go +++ b/cmd/harmony/config_migrations.go @@ -319,6 +319,12 @@ func init() { } migrations["2.5.10"] = func(confTree *toml.Tree) *toml.Tree { + if confTree.Get("P2P.ConnManagerLowWatermark") == nil { + confTree.Set("P2P.ConnManagerLowWatermark", defaultConfig.P2P.ConnManagerLowWatermark) + } + if confTree.Get("P2P.ConnManagerHighWatermark") == nil { + confTree.Set("P2P.ConnManagerHighWatermark", defaultConfig.P2P.ConnManagerHighWatermark) + } if confTree.Get("Sync.MaxAdvertiseWaitTime") == nil { confTree.Set("Sync.MaxAdvertiseWaitTime", defaultConfig.Sync.MaxAdvertiseWaitTime) } diff --git a/cmd/harmony/default.go b/cmd/harmony/default.go index 2a6c7a75c3..78d2643b60 100644 --- a/cmd/harmony/default.go +++ b/cmd/harmony/default.go @@ -32,6 +32,8 @@ var defaultConfig = harmonyconfig.HarmonyConfig{ MaxConnsPerIP: nodeconfig.DefaultMaxConnPerIP, DisablePrivateIPScan: false, MaxPeers: nodeconfig.DefaultMaxPeers, + ConnManagerLowWatermark: nodeconfig.DefaultConnManagerLowWatermark, + ConnManagerHighWatermark: nodeconfig.DefaultConnManagerHighWatermark, WaitForEachPeerToConnect: nodeconfig.DefaultWaitForEachPeerToConnect, }, HTTP: harmonyconfig.HttpConfig{ diff --git a/cmd/harmony/flags.go b/cmd/harmony/flags.go index 168081660c..55da51f551 100644 --- a/cmd/harmony/flags.go +++ b/cmd/harmony/flags.go @@ -63,6 +63,8 @@ var ( p2pDisablePrivateIPScanFlag, maxConnPerIPFlag, maxPeersFlag, + connManagerLowWatermarkFlag, + connManagerHighWatermarkFlag, } httpFlags = []cli.Flag{ @@ -579,6 +581,16 @@ var ( Usage: "maximum number of peers allowed, 0 means no limit", DefValue: defaultConfig.P2P.MaxConnsPerIP, } + connManagerLowWatermarkFlag = cli.IntFlag{ + Name: "p2p.connmgr-low", + Usage: "lowest number of connections that'll be maintained in connection manager", + DefValue: defaultConfig.P2P.ConnManagerLowWatermark, + } + connManagerHighWatermarkFlag = cli.IntFlag{ + Name: "p2p.connmgr-high", + Usage: "highest number of connections that'll be maintained in connection manager", + DefValue: defaultConfig.P2P.ConnManagerHighWatermark, + } waitForEachPeerToConnectFlag = cli.BoolFlag{ Name: "p2p.wait-for-connections", Usage: "node waits for each single peer to connect and it doesn't add them to peers list after timeout", @@ -624,6 +636,14 @@ func applyP2PFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { config.P2P.WaitForEachPeerToConnect = cli.GetBoolFlagValue(cmd, waitForEachPeerToConnectFlag) } + if cli.IsFlagChanged(cmd, connManagerLowWatermarkFlag) { + config.P2P.ConnManagerLowWatermark = cli.GetIntFlagValue(cmd, connManagerLowWatermarkFlag) + } + + if cli.IsFlagChanged(cmd, connManagerHighWatermarkFlag) { + config.P2P.ConnManagerHighWatermark = cli.GetIntFlagValue(cmd, connManagerHighWatermarkFlag) + } + if cli.IsFlagChanged(cmd, p2pDisablePrivateIPScanFlag) { config.P2P.DisablePrivateIPScan = cli.GetBoolFlagValue(cmd, p2pDisablePrivateIPScanFlag) } diff --git a/cmd/harmony/flags_test.go b/cmd/harmony/flags_test.go index 7bd3e71999..9ebae4bb3e 100644 --- a/cmd/harmony/flags_test.go +++ b/cmd/harmony/flags_test.go @@ -65,6 +65,8 @@ func TestHarmonyFlags(t *testing.T) { MaxConnsPerIP: 5, DisablePrivateIPScan: false, MaxPeers: defaultConfig.P2P.MaxPeers, + ConnManagerLowWatermark: defaultConfig.P2P.ConnManagerLowWatermark, + ConnManagerHighWatermark: defaultConfig.P2P.ConnManagerHighWatermark, WaitForEachPeerToConnect: false, }, HTTP: harmonyconfig.HttpConfig{ @@ -374,6 +376,8 @@ func TestP2PFlags(t *testing.T) { MaxConnsPerIP: 10, DisablePrivateIPScan: false, MaxPeers: defaultConfig.P2P.MaxPeers, + ConnManagerLowWatermark: defaultConfig.P2P.ConnManagerLowWatermark, + ConnManagerHighWatermark: defaultConfig.P2P.ConnManagerHighWatermark, WaitForEachPeerToConnect: false, }, }, @@ -386,6 +390,8 @@ func TestP2PFlags(t *testing.T) { MaxConnsPerIP: 10, DisablePrivateIPScan: false, MaxPeers: defaultConfig.P2P.MaxPeers, + ConnManagerLowWatermark: defaultConfig.P2P.ConnManagerLowWatermark, + ConnManagerHighWatermark: defaultConfig.P2P.ConnManagerHighWatermark, WaitForEachPeerToConnect: false, }, }, @@ -399,6 +405,8 @@ func TestP2PFlags(t *testing.T) { MaxConnsPerIP: 5, DisablePrivateIPScan: false, MaxPeers: defaultConfig.P2P.MaxPeers, + ConnManagerLowWatermark: defaultConfig.P2P.ConnManagerLowWatermark, + ConnManagerHighWatermark: defaultConfig.P2P.ConnManagerHighWatermark, WaitForEachPeerToConnect: false, }, }, @@ -412,6 +420,8 @@ func TestP2PFlags(t *testing.T) { MaxConnsPerIP: nodeconfig.DefaultMaxConnPerIP, DisablePrivateIPScan: true, MaxPeers: defaultConfig.P2P.MaxPeers, + ConnManagerLowWatermark: defaultConfig.P2P.ConnManagerLowWatermark, + ConnManagerHighWatermark: defaultConfig.P2P.ConnManagerHighWatermark, WaitForEachPeerToConnect: false, }, }, @@ -425,6 +435,8 @@ func TestP2PFlags(t *testing.T) { MaxConnsPerIP: nodeconfig.DefaultMaxConnPerIP, DisablePrivateIPScan: defaultConfig.P2P.DisablePrivateIPScan, MaxPeers: 100, + ConnManagerLowWatermark: defaultConfig.P2P.ConnManagerLowWatermark, + ConnManagerHighWatermark: defaultConfig.P2P.ConnManagerHighWatermark, WaitForEachPeerToConnect: false, }, }, diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index b68e757422..eacc392f10 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -645,6 +645,8 @@ func createGlobalConfig(hc harmonyconfig.HarmonyConfig) (*nodeconfig.ConfigType, MaxConnPerIP: hc.P2P.MaxConnsPerIP, DisablePrivateIPScan: hc.P2P.DisablePrivateIPScan, MaxPeers: hc.P2P.MaxPeers, + ConnManagerLowWatermark: hc.P2P.ConnManagerLowWatermark, + ConnManagerHighWatermark: hc.P2P.ConnManagerHighWatermark, WaitForEachPeerToConnect: hc.P2P.WaitForEachPeerToConnect, ForceReachabilityPublic: forceReachabilityPublic, }) diff --git a/internal/configs/harmony/harmony.go b/internal/configs/harmony/harmony.go index 73ddd442dd..af3c8c9b61 100644 --- a/internal/configs/harmony/harmony.go +++ b/internal/configs/harmony/harmony.go @@ -54,6 +54,8 @@ type P2pConfig struct { MaxConnsPerIP int DisablePrivateIPScan bool MaxPeers int64 + ConnManagerLowWatermark int + ConnManagerHighWatermark int WaitForEachPeerToConnect bool } diff --git a/internal/configs/node/network.go b/internal/configs/node/network.go index 332b5cce74..8b15d3359c 100644 --- a/internal/configs/node/network.go +++ b/internal/configs/node/network.go @@ -63,7 +63,13 @@ const ( DefaultMaxConnPerIP = 10 // DefaultMaxPeers is the maximum number of remote peers, with 0 representing no limit DefaultMaxPeers = 0 - // DefaultWaitForEachPeerToConnect sets the sync configs to connect to neighbor peers one by one and waits for each peer to connect + // DefaultConnManagerLowWatermark is the lowest number of connections that'll be maintained in connection manager + DefaultConnManagerLowWatermark = 160 + // DefaultConnManagerHighWatermark is the highest number of connections that'll be maintained in connection manager + // When the peer count exceeds the 'high watermark', as many peers will be pruned (and + // their connections terminated) until 'low watermark' peers remain. + DefaultConnManagerHighWatermark = 192 + // DefaultWaitForEachPeerToConnect sets the sync configs to connect to neighbor peers one by one and waits for each peer to connect. DefaultWaitForEachPeerToConnect = false ) diff --git a/node/node_handler.go b/node/node_handler.go index 3db4f8deaf..a34d73e94d 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -3,6 +3,7 @@ package node import ( "bytes" "context" + "fmt" "math/rand" "time" @@ -447,6 +448,7 @@ func (node *Node) BootstrapConsensus() error { if numPeersNow >= min { utils.Logger().Info().Msg("[bootstrap] StartConsensus") enoughMinPeers <- struct{}{} + fmt.Println("Bootstrap consensus done.", numPeersNow, " peers are connected") return } utils.Logger().Info(). diff --git a/p2p/discovery/option_test.go b/p2p/discovery/option_test.go index e782151f59..1829e77c85 100644 --- a/p2p/discovery/option_test.go +++ b/p2p/discovery/option_test.go @@ -40,14 +40,14 @@ func TestDHTOption_getLibp2pRawOptions(t *testing.T) { opt: DHTConfig{ BootNodes: testAddrStr, }, - expLen: 2, + expLen: 1, }, { opt: DHTConfig{ BootNodes: testAddrStr, DataStoreFile: &validPath, }, - expLen: 3, + expLen: 2, }, { opt: DHTConfig{ diff --git a/p2p/host.go b/p2p/host.go index bab1da691c..9395e1df45 100644 --- a/p2p/host.go +++ b/p2p/host.go @@ -96,6 +96,8 @@ type HostConfig struct { MaxConnPerIP int DisablePrivateIPScan bool MaxPeers int64 + ConnManagerLowWatermark int + ConnManagerHighWatermark int WaitForEachPeerToConnect bool ForceReachabilityPublic bool } @@ -114,19 +116,6 @@ func init() { libp2p_pubsub.GossipSubMaxIHaveLength = 1000 } -func forceReachabilityPublic(f bool) libp2p_config.Option { - if f { - return func(cfg *libp2p_config.Config) error { - public := libp2p_network.Reachability(libp2p_network.ReachabilityPublic) - cfg.AutoNATConfig.ForceReachability = &public - return nil - } - } - return func(p2pConfig *libp2p_config.Config) error { - return nil - } -} - // NewHost .. func NewHost(cfg HostConfig) (Host, error) { var ( @@ -141,16 +130,20 @@ func NewHost(cfg HostConfig) (Host, error) { ) ctx, cancel := context.WithCancel(context.Background()) - // TODO: move low and high to configs - connmgr, err := connmgr.NewConnManager( - int(cfg.MaxConnPerIP), // LowWater - int(1024)*cfg.MaxConnPerIP, // HighWater, - connmgr.WithGracePeriod(time.Minute), - ) - if err != nil { + + // create connection manager + low := cfg.ConnManagerLowWatermark + high := cfg.ConnManagerHighWatermark + if high < low { cancel() - return nil, err + utils.Logger().Error(). + Int("low", cfg.ConnManagerLowWatermark). + Int("high", cfg.ConnManagerHighWatermark). + Msg("connection manager watermarks are invalid") + return nil, errors.New("invalid connection manager watermarks") } + + // prepare host options var idht *dht.IpfsDHT var opt discovery.DHTConfig p2pHostConfig := []libp2p.Option{ @@ -164,7 +157,7 @@ func NewHost(cfg HostConfig) (Host, error) { libp2p.DefaultTransports, // Prevent the peer from having too many // connections by attaching a connection manager. - libp2p.ConnectionManager(connmgr), + connectionManager(low, high), // Attempt to open ports using uPNP for NATed hosts. libp2p.NATPortMap(), libp2p.Routing(func(h host.Host) (routing.PeerRouting, error) { @@ -286,6 +279,33 @@ func NewHost(cfg HostConfig) (Host, error) { return h, nil } +// connectionManager creates a new connection manager and configures libp2p to use the +// given connection manager. +// lo and hi are watermarks governing the number of connections that'll be maintained. +// When the peer count exceeds the 'high watermark', as many peers will be pruned (and +// their connections terminated) until 'low watermark' peers remain. +func connectionManager(low int, high int) libp2p_config.Option { + if low > 0 && high > low { + connmgr, err := connmgr.NewConnManager( + low, // Low Watermark + high, // High Watermark + connmgr.WithGracePeriod(time.Minute), + ) + if err != nil { + utils.Logger().Error(). + Err(err). + Int("low", low). + Int("high", high). + Msg("create connection manager failed") + return nil + } + return libp2p.ConnectionManager(connmgr) + } + return func(p2pConfig *libp2p_config.Config) error { + return nil + } +} + // HostV2 is the version 2 p2p host type HostV2 struct { h libp2p_host.Host diff --git a/rosetta/infra/harmony-mainnet.conf b/rosetta/infra/harmony-mainnet.conf index 3e7085ba63..64e78c7eb2 100644 --- a/rosetta/infra/harmony-mainnet.conf +++ b/rosetta/infra/harmony-mainnet.conf @@ -68,6 +68,8 @@ Version = "2.5.11" MaxConnsPerIP = 10 MaxPeers = 0 Port = 9000 + ConnManagerLowWatermark = 160 + ConnManagerHighWatermark = 192 WaitForEachPeerToConnect = false [Pprof] diff --git a/rosetta/infra/harmony-pstn.conf b/rosetta/infra/harmony-pstn.conf index 458e826354..f9ded4d38c 100644 --- a/rosetta/infra/harmony-pstn.conf +++ b/rosetta/infra/harmony-pstn.conf @@ -68,6 +68,8 @@ Version = "2.5.11" MaxConnsPerIP = 10 MaxPeers = 0 Port = 9000 + ConnManagerLowWatermark = 160 + ConnManagerHighWatermark = 192 WaitForEachPeerToConnect = false [Pprof] From 24ba7e75e125a5578c7135b0b34ec7bc851bf740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Mon, 13 Feb 2023 15:33:27 +0800 Subject: [PATCH 053/420] add test for new conn manager flags --- cmd/harmony/flags_test.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/cmd/harmony/flags_test.go b/cmd/harmony/flags_test.go index 9ebae4bb3e..7f8cf34a83 100644 --- a/cmd/harmony/flags_test.go +++ b/cmd/harmony/flags_test.go @@ -440,6 +440,36 @@ func TestP2PFlags(t *testing.T) { WaitForEachPeerToConnect: false, }, }, + { + args: []string{"--p2p.connmgr-low", "100"}, + expConfig: harmonyconfig.P2pConfig{ + Port: nodeconfig.DefaultP2PPort, + IP: nodeconfig.DefaultPublicListenIP, + KeyFile: "./.hmykey", + DiscConcurrency: nodeconfig.DefaultP2PConcurrency, + MaxConnsPerIP: nodeconfig.DefaultMaxConnPerIP, + DisablePrivateIPScan: defaultConfig.P2P.DisablePrivateIPScan, + MaxPeers: defaultConfig.P2P.MaxPeers, + ConnManagerLowWatermark: 100, + ConnManagerHighWatermark: defaultConfig.P2P.ConnManagerHighWatermark, + WaitForEachPeerToConnect: false, + }, + }, + { + args: []string{"--p2p.connmgr-high", "400"}, + expConfig: harmonyconfig.P2pConfig{ + Port: nodeconfig.DefaultP2PPort, + IP: nodeconfig.DefaultPublicListenIP, + KeyFile: "./.hmykey", + DiscConcurrency: nodeconfig.DefaultP2PConcurrency, + MaxConnsPerIP: nodeconfig.DefaultMaxConnPerIP, + DisablePrivateIPScan: defaultConfig.P2P.DisablePrivateIPScan, + MaxPeers: defaultConfig.P2P.MaxPeers, + ConnManagerLowWatermark: defaultConfig.P2P.ConnManagerLowWatermark, + ConnManagerHighWatermark: 400, + WaitForEachPeerToConnect: false, + }, + }, } for i, test := range tests { ts := newFlagTestSuite(t, append(p2pFlags, legacyMiscFlags...), From 76afae993c22a4de7084226dcb6c525ecb1bd10f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Tue, 14 Feb 2023 08:09:08 +0800 Subject: [PATCH 054/420] fix dedent --- rosetta/infra/harmony-mainnet.conf | 2 +- rosetta/infra/harmony-pstn.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rosetta/infra/harmony-mainnet.conf b/rosetta/infra/harmony-mainnet.conf index 64e78c7eb2..8dc7909e3d 100644 --- a/rosetta/infra/harmony-mainnet.conf +++ b/rosetta/infra/harmony-mainnet.conf @@ -69,7 +69,7 @@ Version = "2.5.11" MaxPeers = 0 Port = 9000 ConnManagerLowWatermark = 160 - ConnManagerHighWatermark = 192 + ConnManagerHighWatermark = 192 WaitForEachPeerToConnect = false [Pprof] diff --git a/rosetta/infra/harmony-pstn.conf b/rosetta/infra/harmony-pstn.conf index f9ded4d38c..0835f70d78 100644 --- a/rosetta/infra/harmony-pstn.conf +++ b/rosetta/infra/harmony-pstn.conf @@ -69,7 +69,7 @@ Version = "2.5.11" MaxPeers = 0 Port = 9000 ConnManagerLowWatermark = 160 - ConnManagerHighWatermark = 192 + ConnManagerHighWatermark = 192 WaitForEachPeerToConnect = false [Pprof] From d40697e1c60abee420b127e611a10c8dc4ac9b5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Tue, 14 Feb 2023 09:07:08 +0800 Subject: [PATCH 055/420] add comment to inform about p2p connection manager options --- cmd/harmony/flags.go | 4 ++-- internal/configs/harmony/harmony.go | 19 +++++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/cmd/harmony/flags.go b/cmd/harmony/flags.go index 55da51f551..947adacc5e 100644 --- a/cmd/harmony/flags.go +++ b/cmd/harmony/flags.go @@ -583,12 +583,12 @@ var ( } connManagerLowWatermarkFlag = cli.IntFlag{ Name: "p2p.connmgr-low", - Usage: "lowest number of connections that'll be maintained in connection manager", + Usage: "lowest number of connections that'll be maintained in connection manager. Set both high and low watermarks to zero to disable connection manager", DefValue: defaultConfig.P2P.ConnManagerLowWatermark, } connManagerHighWatermarkFlag = cli.IntFlag{ Name: "p2p.connmgr-high", - Usage: "highest number of connections that'll be maintained in connection manager", + Usage: "highest number of connections that'll be maintained in connection manager. Set both high and low watermarks to zero to disable connection manager", DefValue: defaultConfig.P2P.ConnManagerHighWatermark, } waitForEachPeerToConnectFlag = cli.BoolFlag{ diff --git a/internal/configs/harmony/harmony.go b/internal/configs/harmony/harmony.go index af3c8c9b61..c02f3bb7e0 100644 --- a/internal/configs/harmony/harmony.go +++ b/internal/configs/harmony/harmony.go @@ -46,14 +46,17 @@ type NetworkConfig struct { } type P2pConfig struct { - Port int - IP string - KeyFile string - DHTDataStore *string `toml:",omitempty"` - DiscConcurrency int // Discovery Concurrency value - MaxConnsPerIP int - DisablePrivateIPScan bool - MaxPeers int64 + Port int + IP string + KeyFile string + DHTDataStore *string `toml:",omitempty"` + DiscConcurrency int // Discovery Concurrency value + MaxConnsPerIP int + DisablePrivateIPScan bool + MaxPeers int64 + // In order to disable Connection Manager, it only needs to + // set both the high and low watermarks to zero. In this way, + // using Connection Manager will be an optional feature. ConnManagerLowWatermark int ConnManagerHighWatermark int WaitForEachPeerToConnect bool From 47d2f79b699318a37325ffee9f520414a47646eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Fri, 3 Feb 2023 09:17:49 +0800 Subject: [PATCH 056/420] fix max height issue --- api/service/legacysync/epoch_syncing.go | 22 +++++++++--------- api/service/legacysync/helpers.go | 9 ++++++-- api/service/legacysync/syncing.go | 30 +++++++++++++++++++------ api/service/stagedsync/stagedsync.go | 19 +++++++++++----- api/service/stagedsync/sync_status.go | 5 ++++- 5 files changed, 59 insertions(+), 26 deletions(-) diff --git a/api/service/legacysync/epoch_syncing.go b/api/service/legacysync/epoch_syncing.go index 8d1cb37c4f..eefca9a5cb 100644 --- a/api/service/legacysync/epoch_syncing.go +++ b/api/service/legacysync/epoch_syncing.go @@ -2,7 +2,6 @@ package legacysync import ( "fmt" - "math" "sync" "time" @@ -49,11 +48,12 @@ func (ss *EpochSync) isSynchronized(_ bool) SyncCheckResult { if ss.syncConfig == nil { return SyncCheckResult{} // If syncConfig is not instantiated, return not in sync } - otherHeight1 := getMaxPeerHeight(ss.syncConfig) - if otherHeight1 == math.MaxUint64 { + otherHeight1, errMaxHeight := getMaxPeerHeight(ss.syncConfig) + if errMaxHeight != nil { utils.Logger().Error(). Uint64("OtherHeight", otherHeight1). Int("Peers count", ss.syncConfig.PeersCount()). + Err(errMaxHeight). Msg("[EPOCHSYNC] No peers for get height") return SyncCheckResult{} } @@ -93,15 +93,15 @@ func (ss *EpochSync) SyncLoop(bc core.BlockChain, consensus *consensus.Consensus func syncLoop(bc core.BlockChain, syncConfig *SyncConfig) (timeout int) { isBeacon := bc.ShardID() == 0 - maxHeight := getMaxPeerHeight(syncConfig) - for { - if maxHeight == 0 || maxHeight == math.MaxUint64 { - utils.Logger().Info(). - Msgf("[EPOCHSYNC] No peers to sync (isBeacon: %t, ShardID: %d, peersCount: %d)", - isBeacon, bc.ShardID(), syncConfig.PeersCount()) - return 10 - } + maxHeight, errMaxHeight := getMaxPeerHeight(syncConfig) + if errMaxHeight != nil { + utils.Logger().Info(). + Msgf("[EPOCHSYNC] No peers to sync (isBeacon: %t, ShardID: %d, peersCount: %d)", + isBeacon, bc.ShardID(), syncConfig.PeersCount()) + return 10 + } + for { curEpoch := bc.CurrentBlock().Epoch().Uint64() otherEpoch := shard.Schedule.CalcEpochNumber(maxHeight).Uint64() if otherEpoch == curEpoch+1 { diff --git a/api/service/legacysync/helpers.go b/api/service/legacysync/helpers.go index 4699257f8b..ae1fc34f16 100644 --- a/api/service/legacysync/helpers.go +++ b/api/service/legacysync/helpers.go @@ -14,7 +14,7 @@ import ( ) // getMaxPeerHeight gets the maximum blockchain heights from peers -func getMaxPeerHeight(syncConfig *SyncConfig) uint64 { +func getMaxPeerHeight(syncConfig *SyncConfig) (uint64, error) { maxHeight := uint64(math.MaxUint64) var ( wg sync.WaitGroup @@ -49,7 +49,12 @@ func getMaxPeerHeight(syncConfig *SyncConfig) uint64 { return }) wg.Wait() - return maxHeight + + if maxHeight == uint64(math.MaxUint64) { + return 0, fmt.Errorf("[SYNC] get max peer height failed") + } + + return maxHeight, nil } func createSyncConfig(syncConfig *SyncConfig, peers []p2p.Peer, shardID uint32, selfPeerID libp2p_peer.ID, waitForEachPeerToConnect bool) (*SyncConfig, error) { diff --git a/api/service/legacysync/syncing.go b/api/service/legacysync/syncing.go index 0f004bfb4f..51366ab94c 100644 --- a/api/service/legacysync/syncing.go +++ b/api/service/legacysync/syncing.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "fmt" + "math" "math/rand" "reflect" "sort" @@ -1060,13 +1061,16 @@ func (ss *StateSync) RegisterNodeInfo() int { // IsSameBlockchainHeight checks whether the node is out of sync from other peers func (ss *StateSync) IsSameBlockchainHeight(bc core.BlockChain) (uint64, bool) { - otherHeight := getMaxPeerHeight(ss.syncConfig) + otherHeight, err := getMaxPeerHeight(ss.syncConfig) + if err != nil { + return 0, false + } currentHeight := bc.CurrentBlock().NumberU64() return otherHeight, currentHeight == otherHeight } // GetMaxPeerHeight .. -func (ss *StateSync) GetMaxPeerHeight() uint64 { +func (ss *StateSync) GetMaxPeerHeight() (uint64, error) { return getMaxPeerHeight(ss.syncConfig) } @@ -1079,9 +1083,9 @@ func (ss *StateSync) SyncLoop(bc core.BlockChain, worker *worker.Worker, isBeaco for { start := time.Now() - otherHeight := getMaxPeerHeight(ss.syncConfig) currentHeight := bc.CurrentBlock().NumberU64() - if currentHeight >= otherHeight { + otherHeight, errMaxHeight := getMaxPeerHeight(ss.syncConfig) + if errMaxHeight != nil || currentHeight >= otherHeight { utils.Logger().Info(). Msgf("[SYNC] Node is now IN SYNC! (isBeacon: %t, ShardID: %d, otherHeight: %d, currentHeight: %d)", isBeacon, bc.ShardID(), otherHeight, currentHeight) @@ -1218,7 +1222,9 @@ func (status *syncStatus) Get(fallback func() SyncCheckResult) SyncCheckResult { defer status.lock.Unlock() if status.expired() { result := fallback() - status.update(result) + if result.OtherHeight > 0 && result.OtherHeight < uint64(math.MaxUint64) { + status.update(result) + } } return status.lastResult } @@ -1280,8 +1286,15 @@ func (ss *StateSync) isSynchronized(doubleCheck bool) SyncCheckResult { if ss.syncConfig == nil { return SyncCheckResult{} // If syncConfig is not instantiated, return not in sync } - otherHeight1 := getMaxPeerHeight(ss.syncConfig) lastHeight := ss.blockChain.CurrentBlock().NumberU64() + otherHeight1, errMaxHeight1 := getMaxPeerHeight(ss.syncConfig) + if errMaxHeight1 != nil { + return SyncCheckResult{ + IsSynchronized: false, + OtherHeight: 0, + HeightDiff: 0, + } + } wasOutOfSync := lastHeight+inSyncThreshold < otherHeight1 if !doubleCheck { @@ -1302,7 +1315,10 @@ func (ss *StateSync) isSynchronized(doubleCheck bool) SyncCheckResult { // double check the sync status after 1 second to confirm (avoid false alarm) time.Sleep(1 * time.Second) - otherHeight2 := getMaxPeerHeight(ss.syncConfig) + otherHeight2, errMaxHeight2 := getMaxPeerHeight(ss.syncConfig) + if errMaxHeight2 != nil { + otherHeight2 = otherHeight1 + } currentHeight := ss.blockChain.CurrentBlock().NumberU64() isOutOfSync := currentHeight+inSyncThreshold < otherHeight2 diff --git a/api/service/stagedsync/stagedsync.go b/api/service/stagedsync/stagedsync.go index c81f0b5b12..aca707f619 100644 --- a/api/service/stagedsync/stagedsync.go +++ b/api/service/stagedsync/stagedsync.go @@ -1201,9 +1201,8 @@ func (ss *StagedSync) IsSameBlockchainHeight(bc core.BlockChain) (uint64, bool) } // GetMaxPeerHeight returns maximum block height of connected peers -func (ss *StagedSync) GetMaxPeerHeight() uint64 { - mph, _ := ss.getMaxPeerHeight() - return mph +func (ss *StagedSync) GetMaxPeerHeight() (uint64, error) { + return ss.getMaxPeerHeight() } func (ss *StagedSync) addConsensusLastMile(bc core.BlockChain, consensus *consensus.Consensus) error { @@ -1277,8 +1276,15 @@ func (ss *StagedSync) isSynchronized(doubleCheck bool) SyncCheckResult { if ss.syncConfig == nil { return SyncCheckResult{} // If syncConfig is not instantiated, return not in sync } - otherHeight1, _ := ss.getMaxPeerHeight() lastHeight := ss.Blockchain().CurrentBlock().NumberU64() + otherHeight1, errMaxHeight1 := ss.getMaxPeerHeight() + if errMaxHeight1 != nil { + return SyncCheckResult{ + IsSynchronized: false, + OtherHeight: 0, + HeightDiff: 0, + } + } wasOutOfSync := lastHeight+inSyncThreshold < otherHeight1 if !doubleCheck { @@ -1299,7 +1305,10 @@ func (ss *StagedSync) isSynchronized(doubleCheck bool) SyncCheckResult { // double check the sync status after 1 second to confirm (avoid false alarm) time.Sleep(1 * time.Second) - otherHeight2, _ := ss.getMaxPeerHeight() + otherHeight2, errMaxHeight2 := ss.getMaxPeerHeight() + if errMaxHeight2 != nil { + otherHeight2 = otherHeight1 + } currentHeight := ss.Blockchain().CurrentBlock().NumberU64() isOutOfSync := currentHeight+inSyncThreshold < otherHeight2 diff --git a/api/service/stagedsync/sync_status.go b/api/service/stagedsync/sync_status.go index 556f1058ba..3f1bf11e89 100644 --- a/api/service/stagedsync/sync_status.go +++ b/api/service/stagedsync/sync_status.go @@ -1,6 +1,7 @@ package stagedsync import ( + "math" "sync" "time" @@ -75,7 +76,9 @@ func (status *syncStatus) Get(fallback func() SyncCheckResult) SyncCheckResult { defer status.lock.Unlock() if status.expired() { result := fallback() - status.update(result) + if result.OtherHeight > 0 && result.OtherHeight < uint64(math.MaxUint64) { + status.update(result) + } } return status.lastResult } From 8379ce90100d626217a7806b94ea294e83712dd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Fri, 3 Feb 2023 17:04:14 +0800 Subject: [PATCH 057/420] add a separate log for get max height error --- api/service/legacysync/syncing.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/api/service/legacysync/syncing.go b/api/service/legacysync/syncing.go index 51366ab94c..43f07c039b 100644 --- a/api/service/legacysync/syncing.go +++ b/api/service/legacysync/syncing.go @@ -1085,7 +1085,16 @@ func (ss *StateSync) SyncLoop(bc core.BlockChain, worker *worker.Worker, isBeaco start := time.Now() currentHeight := bc.CurrentBlock().NumberU64() otherHeight, errMaxHeight := getMaxPeerHeight(ss.syncConfig) - if errMaxHeight != nil || currentHeight >= otherHeight { + if errMaxHeight != nil { + utils.Logger().Error(). + Bool("isBeacon", isBeacon). + Uint32("ShardID", bc.ShardID()). + Uint64("currentHeight", currentHeight). + Int("peers count", ss.syncConfig.PeersCount()). + Msgf("[SYNC] get max height failed") + break + } + if currentHeight >= otherHeight { utils.Logger().Info(). Msgf("[SYNC] Node is now IN SYNC! (isBeacon: %t, ShardID: %d, otherHeight: %d, currentHeight: %d)", isBeacon, bc.ShardID(), otherHeight, currentHeight) From e795f457d0b88d34313fd585d82feb727d463699 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Fri, 10 Feb 2023 22:59:05 +0800 Subject: [PATCH 058/420] fix log --- api/service/legacysync/helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/service/legacysync/helpers.go b/api/service/legacysync/helpers.go index ae1fc34f16..14dac994d9 100644 --- a/api/service/legacysync/helpers.go +++ b/api/service/legacysync/helpers.go @@ -51,7 +51,7 @@ func getMaxPeerHeight(syncConfig *SyncConfig) (uint64, error) { wg.Wait() if maxHeight == uint64(math.MaxUint64) { - return 0, fmt.Errorf("[SYNC] get max peer height failed") + return 0, fmt.Errorf("get max peer height failed") } return maxHeight, nil From 0d994276c2693d4e8c9a669323c1cc1f8e7fd302 Mon Sep 17 00:00:00 2001 From: Sun Hyuk Ahn Date: Thu, 9 Feb 2023 16:46:46 -0800 Subject: [PATCH 059/420] feat: triesInMemory flag --- cmd/harmony/config_migrations.go | 8 +++ cmd/harmony/default.go | 3 +- cmd/harmony/flags.go | 14 +++++ cmd/harmony/flags_test.go | 88 +++++++++++++++++------------ core/blockchain_impl.go | 19 +++---- internal/configs/harmony/harmony.go | 1 + internal/shardchain/shardchains.go | 10 ++++ rosetta/infra/harmony-mainnet.conf | 1 + rosetta/infra/harmony-pstn.conf | 1 + 9 files changed, 97 insertions(+), 48 deletions(-) diff --git a/cmd/harmony/config_migrations.go b/cmd/harmony/config_migrations.go index bdbfefdc91..7c9420a0be 100644 --- a/cmd/harmony/config_migrations.go +++ b/cmd/harmony/config_migrations.go @@ -332,6 +332,14 @@ func init() { return confTree } + migrations["2.5.11"] = func(confTree *toml.Tree) *toml.Tree { + if confTree.Get("General.TriesInMemory") == nil { + confTree.Set("General.TriesInMemory", defaultConfig.General.TriesInMemory) + } + confTree.Set("Version", "2.5.12") + return confTree + } + // check that the latest version here is the same as in default.go largestKey := getNextVersion(migrations) if largestKey != tomlConfigVersion { diff --git a/cmd/harmony/default.go b/cmd/harmony/default.go index 78d2643b60..3835c49210 100644 --- a/cmd/harmony/default.go +++ b/cmd/harmony/default.go @@ -5,7 +5,7 @@ import ( nodeconfig "github.com/harmony-one/harmony/internal/configs/node" ) -const tomlConfigVersion = "2.5.11" +const tomlConfigVersion = "2.5.12" const ( defNetworkType = nodeconfig.Mainnet @@ -22,6 +22,7 @@ var defaultConfig = harmonyconfig.HarmonyConfig{ IsOffline: false, DataDir: "./", TraceEnable: false, + TriesInMemory: 128, }, Network: getDefaultNetworkConfig(defNetworkType), P2P: harmonyconfig.P2pConfig{ diff --git a/cmd/harmony/flags.go b/cmd/harmony/flags.go index 947adacc5e..c3b03e336f 100644 --- a/cmd/harmony/flags.go +++ b/cmd/harmony/flags.go @@ -31,6 +31,7 @@ var ( legacyDataDirFlag, taraceFlag, + triesInMemoryFlag, } dnsSyncFlags = []cli.Flag{ @@ -322,6 +323,11 @@ var ( Usage: "indicates if full transaction tracing should be enabled", DefValue: defaultConfig.General.TraceEnable, } + triesInMemoryFlag = cli.IntFlag{ + Name: "blockchain.tries_in_memory", + Usage: "number of blocks from header stored in disk before exiting", + DefValue: defaultConfig.General.TriesInMemory, + } ) func getRootFlags() []cli.Flag { @@ -399,6 +405,14 @@ func applyGeneralFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) if cli.IsFlagChanged(cmd, isBackupFlag) { config.General.IsBackup = cli.GetBoolFlagValue(cmd, isBackupFlag) } + + if cli.IsFlagChanged(cmd, triesInMemoryFlag) { + value := cli.GetIntFlagValue(cmd, triesInMemoryFlag) + if value <= 1 { + panic("Must number greater than 1 for txpool.accountslots") + } + config.General.TriesInMemory = value + } } // network flags diff --git a/cmd/harmony/flags_test.go b/cmd/harmony/flags_test.go index 7f8cf34a83..15ba9aaf08 100644 --- a/cmd/harmony/flags_test.go +++ b/cmd/harmony/flags_test.go @@ -35,11 +35,12 @@ func TestHarmonyFlags(t *testing.T) { expConfig: harmonyconfig.HarmonyConfig{ Version: tomlConfigVersion, General: harmonyconfig.GeneralConfig{ - NodeType: "validator", - NoStaking: false, - ShardID: -1, - IsArchival: false, - DataDir: "./", + NodeType: "validator", + NoStaking: false, + ShardID: -1, + IsArchival: false, + DataDir: "./", + TriesInMemory: 128, }, Network: harmonyconfig.NetworkConfig{ NetworkType: "mainnet", @@ -186,63 +187,80 @@ func TestGeneralFlags(t *testing.T) { { args: []string{}, expConfig: harmonyconfig.GeneralConfig{ - NodeType: "validator", - NoStaking: false, - ShardID: -1, - IsArchival: false, - DataDir: "./", + NodeType: "validator", + NoStaking: false, + ShardID: -1, + IsArchival: false, + DataDir: "./", + TriesInMemory: 128, }, }, { args: []string{"--run", "explorer", "--run.legacy", "--run.shard=0", "--run.archive=true", "--datadir=./.hmy"}, expConfig: harmonyconfig.GeneralConfig{ - NodeType: "explorer", - NoStaking: true, - ShardID: 0, - IsArchival: true, - DataDir: "./.hmy", + NodeType: "explorer", + NoStaking: true, + ShardID: 0, + IsArchival: true, + DataDir: "./.hmy", + TriesInMemory: 128, }, }, { args: []string{"--node_type", "explorer", "--staking", "--shard_id", "0", "--is_archival", "--db_dir", "./"}, expConfig: harmonyconfig.GeneralConfig{ - NodeType: "explorer", - NoStaking: false, - ShardID: 0, - IsArchival: true, - DataDir: "./", + NodeType: "explorer", + NoStaking: false, + ShardID: 0, + IsArchival: true, + DataDir: "./", + TriesInMemory: 128, }, }, { args: []string{"--staking=false", "--is_archival=false"}, expConfig: harmonyconfig.GeneralConfig{ - NodeType: "validator", - NoStaking: true, - ShardID: -1, - IsArchival: false, - DataDir: "./", + NodeType: "validator", + NoStaking: true, + ShardID: -1, + IsArchival: false, + DataDir: "./", + TriesInMemory: 128, }, }, { args: []string{"--run", "explorer", "--run.shard", "0"}, expConfig: harmonyconfig.GeneralConfig{ - NodeType: "explorer", - NoStaking: false, - ShardID: 0, - IsArchival: false, - DataDir: "./", + NodeType: "explorer", + NoStaking: false, + ShardID: 0, + IsArchival: false, + DataDir: "./", + TriesInMemory: 128, }, }, { args: []string{"--run", "explorer", "--run.shard", "0", "--run.archive=false"}, expConfig: harmonyconfig.GeneralConfig{ - NodeType: "explorer", - NoStaking: false, - ShardID: 0, - IsArchival: false, - DataDir: "./", + NodeType: "explorer", + NoStaking: false, + ShardID: 0, + IsArchival: false, + DataDir: "./", + TriesInMemory: 128, + }, + }, + { + args: []string{"--blockchain.tries_in_memory", "64"}, + expConfig: harmonyconfig.GeneralConfig{ + NodeType: "validator", + NoStaking: false, + ShardID: -1, + IsArchival: false, + DataDir: "./", + TriesInMemory: 64, }, }, } diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 489fb3661e..01f1b21e65 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -103,7 +103,6 @@ const ( maxFutureBlocks = 16 maxTimeFutureBlocks = 30 badBlockLimit = 10 - triesInMemory = 128 triesInRedis = 1000 shardCacheLimit = 10 commitsCacheLimit = 10 @@ -127,6 +126,7 @@ type CacheConfig struct { Disabled bool // Whether to disable trie write caching (archive node) TrieNodeLimit int // Memory limit (MB) at which to flush the current in-memory trie to disk TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk + TriesInMemory uint64 // Block number from the head stored in disk before exiting } type BlockChainImpl struct { @@ -223,12 +223,7 @@ func newBlockChainWithOptions( db ethdb.Database, stateCache state.Database, beaconChain BlockChain, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus_engine.Engine, vmConfig vm.Config, options Options) (*BlockChainImpl, error) { - if cacheConfig == nil { - cacheConfig = &CacheConfig{ - TrieNodeLimit: 256 * 1024 * 1024, - TrieTimeLimit: 2 * time.Minute, - } - } + bodyCache, _ := lru.New(bodyCacheLimit) bodyRLPCache, _ := lru.New(bodyCacheLimit) receiptsCache, _ := lru.New(receiptsCacheLimit) @@ -1080,7 +1075,7 @@ func (bc *BlockChainImpl) Stop() { if !bc.cacheConfig.Disabled { triedb := bc.stateCache.TrieDB() - for _, offset := range []uint64{0, 1, triesInMemory - 1} { + for _, offset := range []uint64{0, 1, bc.cacheConfig.TriesInMemory - 1} { if number := bc.CurrentBlock().NumberU64(); number > offset { recent := bc.GetHeaderByNumber(number - offset) if recent != nil { @@ -1407,7 +1402,7 @@ func (bc *BlockChainImpl) WriteBlockWithState( triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive bc.triegc.Push(root, -int64(block.NumberU64())) - if current := block.NumberU64(); current > triesInMemory { + if current := block.NumberU64(); current > bc.cacheConfig.TriesInMemory { // If we exceeded our memory allowance, flush matured singleton nodes to disk var ( nodes, imgs = triedb.Size() @@ -1417,7 +1412,7 @@ func (bc *BlockChainImpl) WriteBlockWithState( triedb.Cap(limit - ethdb.IdealBatchSize) } // Find the next state trie we need to commit - header := bc.GetHeaderByNumber(current - triesInMemory) + header := bc.GetHeaderByNumber(current - bc.cacheConfig.TriesInMemory) if header != nil { chosen := header.Number().Uint64() @@ -1425,11 +1420,11 @@ func (bc *BlockChainImpl) WriteBlockWithState( if bc.gcproc > bc.cacheConfig.TrieTimeLimit { // If we're exceeding limits but haven't reached a large enough memory gap, // warn the user that the system is becoming unstable. - if chosen < lastWrite+triesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { + if chosen < lastWrite+bc.cacheConfig.TriesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { utils.Logger().Info(). Dur("time", bc.gcproc). Dur("allowance", bc.cacheConfig.TrieTimeLimit). - Float64("optimum", float64(chosen-lastWrite)/triesInMemory). + Float64("optimum", float64(chosen-lastWrite)/float64(bc.cacheConfig.TriesInMemory)). Msg("State in memory for too long, committing") } // Flush an entire trie and restart the counters diff --git a/internal/configs/harmony/harmony.go b/internal/configs/harmony/harmony.go index c02f3bb7e0..67c29f820f 100644 --- a/internal/configs/harmony/harmony.go +++ b/internal/configs/harmony/harmony.go @@ -74,6 +74,7 @@ type GeneralConfig struct { TraceEnable bool EnablePruneBeaconChain bool RunElasticMode bool + TriesInMemory int } type TiKVConfig struct { diff --git a/internal/shardchain/shardchains.go b/internal/shardchain/shardchains.go index 8f9b185956..1be2c68414 100644 --- a/internal/shardchain/shardchains.go +++ b/internal/shardchain/shardchains.go @@ -3,6 +3,7 @@ package shardchain import ( "math/big" "sync" + "time" "github.com/harmony-one/harmony/core/state" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" @@ -104,6 +105,15 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c utils.Logger().Info(). Uint32("shardID", shardID). Msg("disable cache, running in archival mode") + } else { + cacheConfig = &core.CacheConfig{ + TrieNodeLimit: 256 * 1024 * 1024, + TrieTimeLimit: 2 * time.Minute, + TriesInMemory: 128, + } + if sc.harmonyconfig != nil { + cacheConfig.TriesInMemory = uint64(sc.harmonyconfig.General.TriesInMemory) + } } chainConfig := *sc.chainConfig diff --git a/rosetta/infra/harmony-mainnet.conf b/rosetta/infra/harmony-mainnet.conf index 8dc7909e3d..a929eeeab1 100644 --- a/rosetta/infra/harmony-mainnet.conf +++ b/rosetta/infra/harmony-mainnet.conf @@ -35,6 +35,7 @@ Version = "2.5.11" RunElasticMode = false ShardID = 0 TraceEnable = false + TriesInMemory = 128 [HTTP] AuthPort = 9501 diff --git a/rosetta/infra/harmony-pstn.conf b/rosetta/infra/harmony-pstn.conf index 0835f70d78..edb911f87a 100644 --- a/rosetta/infra/harmony-pstn.conf +++ b/rosetta/infra/harmony-pstn.conf @@ -35,6 +35,7 @@ Version = "2.5.11" RunElasticMode = false ShardID = 0 TraceEnable = false + TriesInMemory = 128 [HTTP] AuthPort = 9501 From 7affda48a1179e76cde1e64e949e227326b2b5db Mon Sep 17 00:00:00 2001 From: Sun Hyuk Ahn Date: Fri, 10 Feb 2023 10:56:17 -0800 Subject: [PATCH 060/420] fix: panic if TriesInMemory is 1 to 2 --- cmd/harmony/flags.go | 4 ++-- core/blockchain_impl.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/harmony/flags.go b/cmd/harmony/flags.go index c3b03e336f..2ffceb6e5b 100644 --- a/cmd/harmony/flags.go +++ b/cmd/harmony/flags.go @@ -408,8 +408,8 @@ func applyGeneralFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) if cli.IsFlagChanged(cmd, triesInMemoryFlag) { value := cli.GetIntFlagValue(cmd, triesInMemoryFlag) - if value <= 1 { - panic("Must number greater than 1 for txpool.accountslots") + if value <= 2 { + panic("Must provide number greater than 2 for General.TriesInMemory") } config.General.TriesInMemory = value } diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 01f1b21e65..8b0683bd65 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -1071,7 +1071,7 @@ func (bc *BlockChainImpl) Stop() { // We're writing three different states to catch different restart scenarios: // - HEAD: So we don't need to reprocess any blocks in the general case // - HEAD-1: So we don't do large reorgs if our HEAD becomes an uncle - // - HEAD-127: So we have a hard limit on the number of blocks reexecuted + // - HEAD-TriesInMemory: So we have a configurable hard limit on the number of blocks reexecuted (default 128) if !bc.cacheConfig.Disabled { triedb := bc.stateCache.TrieDB() From 8bd505e78f3e695483b0644fb3dfee1e3ea582f0 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 1 Sep 2022 00:53:16 +0700 Subject: [PATCH 061/420] in progress. --- api/service/explorer/service.go | 48 +++++++++++++++++++++++++++++++ cmd/harmony/main.go | 6 +++- consensus/consensus.go | 2 ++ consensus/consensus_service.go | 27 +++++++++++++++++ consensus/consensus_v2.go | 31 ++++++++++++++++++-- consensus/leader.go | 9 ++++++ consensus/quorum/quorum.go | 7 ++++- consensus/validator.go | 14 +++++++++ consensus/view_change.go | 19 ++++++++++++ hmy/hmy.go | 2 ++ node/api.go | 6 ++++ node/node_newblock.go | 4 +++ test/configs/local-resharding.txt | 31 ++++++-------------- test/deploy.sh | 5 +++- 14 files changed, 184 insertions(+), 27 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index b5ffa1b8b0..d96c1bc56c 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -7,11 +7,13 @@ import ( "fmt" "net" "net/http" + "os" "path" "strconv" "time" "github.com/RoaringBitmap/roaring/roaring64" + "github.com/harmony-one/harmony/internal/common" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" ethCommon "github.com/ethereum/go-ethereum/common" @@ -113,12 +115,17 @@ func (s *Service) Run() *http.Server { s.router = mux.NewRouter() + fmt.Println("++", addr) + // Set up router for addresses. // Fetch addresses request, accepts parameter size: how much addresses to read, // parameter prefix: from which address prefix start s.router.Path("/addresses").Queries("size", "{[0-9]*?}", "prefix", "{[a-zA-Z0-9]*?}").HandlerFunc(s.GetAddresses).Methods("GET") s.router.Path("/addresses").HandlerFunc(s.GetAddresses) s.router.Path("/height").HandlerFunc(s.GetHeight) + s.router.Path("/leader").HandlerFunc(s.GetLeader) + s.router.Path("/blocks").HandlerFunc(s.GetBlocks) + s.router.Path("/halt").HandlerFunc(s.halt) // Set up router for supply info s.router.Path("/burn-addresses").Queries().HandlerFunc(s.GetInaccessibleAddressInfo).Methods("GET") @@ -186,6 +193,47 @@ type HeightResponse struct { S3 uint64 `json:"3,omitempty"` } +func (s *Service) GetLeader(w http.ResponseWriter, r *http.Request) { + if s.backend.IsCurrentlyLeader() { + w.Write([]byte("true ")) + } else { + w.Write([]byte("false")) + } + + keys := "" + for _, p := range s.backend.GetPublicKeys() { + addr := common.Address{} + addrBytes := p.Object.GetAddress() + addr.SetBytes(addrBytes[:]) + keys += fmt.Sprintf("%s ", addr.String()) + break + } + //blsPubKeyBytes := leaderKey.Object.GetAddress() + //coinbase.SetBytes(blsPubKeyBytes[:]) + + w.Write([]byte(fmt.Sprintf(" %d", s.blockchain.ShardID()))) + w.Write([]byte(fmt.Sprintf(" %s", s.Port))) + w.Write([]byte(fmt.Sprintf(" %s", keys))) + w.Write([]byte(fmt.Sprintf(" %s", s.backend.GetPublicKeys().SerializeToHexStr()))) + +} + +func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { + cur := s.blockchain.CurrentHeader().Number().Uint64() + + for i := cur; i > 0; i-- { + block := s.blockchain.GetBlockByNumber(i) + w.Write([]byte(fmt.Sprintf("%d ", i))) + w.Write([]byte(fmt.Sprintf("%s ", block.Header().ViewID().String()))) + w.Write([]byte(fmt.Sprintf("%s ", block.Header().Coinbase().Hash().Hex()))) + w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) + } +} + +func (s *Service) halt(w http.ResponseWriter, r *http.Request) { + os.Exit(0) +} + // GetHeight returns heights of current and beacon chains if needed. func (s *Service) GetHeight(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index eacc392f10..2fcfe5f037 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -266,6 +266,8 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { var err error + fmt.Println("OS: ", os.Args) + nodeconfigSetShardSchedule(hc) nodeconfig.SetShardingSchedule(shard.Schedule) nodeconfig.SetVersion(getHarmonyVersion()) @@ -425,8 +427,9 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { if currentNode.NodeConfig.Role() == nodeconfig.Validator { currentNode.RegisterValidatorServices() } else if currentNode.NodeConfig.Role() == nodeconfig.ExplorerNode { - currentNode.RegisterExplorerServices() + } + currentNode.RegisterExplorerServices() currentNode.RegisterService(service.CrosslinkSending, crosslink_sending.New(currentNode, currentNode.Blockchain())) if hc.Pprof.Enabled { setupPprofService(currentNode, hc) @@ -800,6 +803,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // Set the consensus ID to be the current block number viewID := currentNode.Blockchain().CurrentBlock().Header().ViewID().Uint64() + fmt.Println("viewID:", viewID) currentConsensus.SetViewIDs(viewID + 1) utils.Logger().Info(). Uint64("viewID", viewID). diff --git a/consensus/consensus.go b/consensus/consensus.go index e0e096e306..0c51d648c0 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -73,6 +73,8 @@ type Consensus struct { priKey multibls.PrivateKeys // the publickey of leader LeaderPubKey *bls.PublicKeyWrapper + // index of leader in the list of validators. + LeaderIndex int // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e6be606f2e..e646504dde 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync/atomic" "time" @@ -212,6 +213,7 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } + fmt.Println("[checkViewID] Set LEADEER PUB KEY ", msg.SenderPubkeys[0].Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() @@ -395,6 +397,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { Str("leaderPubKey", leaderPubKey.Bytes.Hex()). Msg("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain") consensus.pubKeyLock.Lock() + fmt.Println("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain", leaderPubKey.Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = leaderPubKey consensus.pubKeyLock.Unlock() } @@ -461,6 +464,7 @@ func (consensus *Consensus) isLeader() bool { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { + fmt.Println("SetViewIDs", height) consensus.SetCurBlockViewID(height) consensus.SetViewChangingID(height) } @@ -470,6 +474,20 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } +// SetLeaderIndex set the leader index. +func (consensus *Consensus) SetLeaderIndex(f func(int) int) (current int) { + consensus.pubKeyLock.Lock() + defer consensus.pubKeyLock.Unlock() + consensus.LeaderIndex = f(consensus.LeaderIndex) + return consensus.LeaderIndex +} + +func (consensus *Consensus) GetLeaderIndex() int { + consensus.pubKeyLock.Lock() + defer consensus.pubKeyLock.Unlock() + return consensus.LeaderIndex +} + // SetViewChangingID set the current view change ID func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) @@ -480,6 +498,15 @@ func (consensus *Consensus) StartFinalityCount() { consensus.finalityCounter.Store(time.Now().UnixNano()) } +//func (consensus *Consensus) ReshardingNextLeader(newblock *types.Block) { +// consensus.pubKeyLock.Lock() +// fmt.Println("nextBlock1 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) +// consensus.LeaderPubKey = consensus.getNextLeaderKey(consensus.GetCurBlockViewID() + 1) +// fmt.Println("nextBlock2 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) +// consensus.pubKeyLock.Unlock() +// +//} + // FinishFinalityCount calculate the current finality func (consensus *Consensus) FinishFinalityCount() { d := time.Now().UnixNano() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 0b1ce28cf4..bf042c62d3 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/hex" + "fmt" "math/big" "sync/atomic" "time" @@ -131,6 +132,7 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb } func (consensus *Consensus) finalCommit() { + // THIS IS NOT GOOD PLACE FOR LEADER SWITCHING numCommits := consensus.Decider.SignersCount(quorum.Commit) consensus.getLogger().Info(). @@ -393,6 +395,7 @@ func (consensus *Consensus) Start( consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() case newBlock := <-blockChannel: + //consensus.ReshardingNextLeader(newBlock) consensus.getLogger().Info(). Uint64("MsgBlockNum", newBlock.NumberU64()). Msg("[ConsensusMainLoop] Received Proposed New Block!") @@ -404,6 +407,7 @@ func (consensus *Consensus) Start( } // Sleep to wait for the full block time consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") + <-time.After(time.Until(consensus.NextBlockDue)) consensus.StartFinalityCount() @@ -528,6 +532,7 @@ func (consensus *Consensus) getLastMileBlocksAndMsg(bnStart uint64) ([]*types.Bl // preCommitAndPropose commit the current block with 67% commit signatures and start // proposing new block which will wait on the full commit signatures to finish func (consensus *Consensus) preCommitAndPropose(blk *types.Block) error { + //fmt.Println("preCommitAndPropose", utils.GetPort(), blk.NumberU64()) if blk == nil { return errors.New("block to pre-commit is nil") } @@ -645,6 +650,7 @@ func (consensus *Consensus) tryCatchup() error { consensus.getLogger().Error().Err(err).Msg("[TryCatchup] Failed to add block to chain") return err } + //fmt.Println("tryCatchup ", utils.GetPort(), blk.NumberU64()) select { // TODO: Remove this when removing dns sync and stream sync is fully up case consensus.VerifiedNewBlock <- blk: @@ -659,6 +665,8 @@ func (consensus *Consensus) tryCatchup() error { } func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMessage) error { + // this function evaluates for all, leader and validators. + if consensus.Blockchain().CurrentBlock().NumberU64() < blk.NumberU64() { if _, err := consensus.Blockchain().InsertChain([]*types.Block{blk}, !consensus.FBFTLog.IsBlockVerified(blk.Hash())); err != nil { consensus.getLogger().Error().Err(err).Msg("[commitBlock] Failed to add block to chain") @@ -727,10 +735,29 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { // SetupForNewConsensus sets the state for new consensus func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) - consensus.SetCurBlockViewID(committedMsg.ViewID + 1) + curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) // first view id is going to be 2. + prev := consensus.GetLeaderPubKey() + idx := consensus.SetLeaderIndex(func(i int) int { + if curBlockViewID%3 == 0 { + return i + 1 + } + return i + }) + pps := consensus.Decider.Participants() consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] + consensus.LeaderPubKey = &pps[idx%len(pps)] + fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %q\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) consensus.pubKeyLock.Unlock() + if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { + // leader changed + go func() { + fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + consensus.ReadySignal <- SyncProposal + + }() + + } var epoch *big.Int if blk.IsLastBlockInEpoch() { epoch = new(big.Int).Add(blk.Epoch(), common.Big1) diff --git a/consensus/leader.go b/consensus/leader.go index 422508762b..a359c229a4 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -199,9 +199,17 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { + // TODO HERE + //if recvMsg.ViewID == 10 { + // return + //} consensus.mutex.Lock() defer consensus.mutex.Unlock() //// Read - Start + if consensus.ShardID == 0 { + //fmt.Println("onCommit ", recvMsg.BlockNum) + } + if !consensus.isRightBlockNumAndViewID(recvMsg) { return } @@ -333,4 +341,5 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED) } + //fmt.Println("onCommit99: ", utils.GetPort(), recvMsg.BlockNum) } diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 6cf79e549a..0586ace4fe 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -230,12 +230,17 @@ func (s *cIdentities) NthNextHmy(instance shardingconfig.Instance, pubKey *bls.P Msg("[NthNextHmy] pubKey not found") } numNodes := instance.NumHarmonyOperatedNodesPerShard() + //fmt.Println("??idx:", idx, numNodes) // sanity check to avoid out of bound access if numNodes <= 0 || numNodes > len(s.publicKeys) { numNodes = len(s.publicKeys) } idx = (idx + next) % numNodes - return found, &s.publicKeys[idx] + //fmt.Println("-------idx:", idx) + + new := &s.publicKeys[idx] + fmt.Println("NthNextHmy: ", pubKey.Bytes.Hex(), new.Bytes.Hex()) + return found, new } // NthNextHmyExt return the Nth next pubkey of Harmony + allowlist nodes, next can be negative number diff --git a/consensus/validator.go b/consensus/validator.go index a73ac92eb3..013769395c 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -18,6 +18,7 @@ import ( ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { + recvMsg, err := consensus.ParseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). @@ -26,6 +27,9 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Msg("[OnAnnounce] Unparseable leader message") return } + if consensus.ShardID == 0 { + //fmt.Println("onAnnounce called ", recvMsg.BlockNum) + } // NOTE let it handle its own logs if !consensus.onAnnounceSanityChecks(recvMsg) { @@ -184,6 +188,9 @@ func (consensus *Consensus) sendCommitMessages(blockObj *types.Block) { // if onPrepared accepts the prepared message from the leader, then // it will send a COMMIT message for the leader to receive on the network. func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { + if consensus.ShardID == 0 { + //fmt.Println("onPrepared", recvMsg.BlockNum) + } consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -399,6 +406,13 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { consensus.getLogger().Info().Msg("[OnCommitted] Start consensus timer (new block added)") consensus.consensusTimeout[timeoutConsensus].Start() } + + //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum) + if blk != nil { + //consensus.ReshardingNextLeader(blk) + } else { + //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum, "blk is nil") + } } // Collect private keys that are part of the current committee. diff --git a/consensus/view_change.go b/consensus/view_change.go index 6631051dcc..fd9cde6c5f 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync" "time" @@ -161,6 +162,7 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { Uint64("stuckBlockViewID", stuckBlockViewID). Msg("[getNextViewID]") + fmt.Println("end getNextViewID: ", nextViewID, viewChangeDuration) // duration is always the fixed view change duration for synchronous view change return nextViewID, viewChangeDuration } @@ -233,6 +235,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe lastLeaderPubKey, gap) } + fmt.Println("wasfoundNext", consensus.Blockchain.Config().IsAllowlistEpoch(epoch), wasFound, next.Bytes.Hex(), lastLeaderPubKey.Bytes.Hex()) if !wasFound { consensus.getLogger().Warn(). Str("key", consensus.LeaderPubKey.Bytes.Hex()). @@ -254,6 +257,7 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { + fmt.Printf("Message to send leader111: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } @@ -264,6 +268,7 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() + //fmt.Println("startViewChange", nextViewID) consensus.SetViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet @@ -307,7 +312,9 @@ func (consensus *Consensus) startViewChange() { if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { continue } + // Тут уже другой leader msgToSend := consensus.constructViewChangeMessage(&key) + fmt.Println("Message to send leader222: ", consensus.LeaderPubKey.Bytes.Hex()) if err := consensus.msgSender.SendWithRetry( consensus.BlockNum(), msg_pb.MessageType_VIEWCHANGE, @@ -365,6 +372,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.ResetState() } + fmt.Println("[startNewView]", newLeaderPriKey.Pub.Bytes.Hex()) consensus.SetLeaderPubKey(newLeaderPriKey.Pub) return nil @@ -372,6 +380,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri // onViewChange is called when the view change message is received. func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { + //fmt.Printf("[onViewChange] received view change message from %+v\n", recvMsg) consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -394,6 +403,13 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { return } + consensus.getLogger().Debug(). + Err(err). + Interface("SenderPubkeys", recvMsg.SenderPubkeys). + Str("NextLeader", recvMsg.LeaderPubkey.Bytes.Hex()). + Str("myBLSPubKey", consensus.priKey.GetPublicKeys().SerializeToHexStr()). + Msg("[onViewChange] I am the Leader") + if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) { consensus.getLogger().Info(). Int64("have", consensus.Decider.SignersCount(quorum.ViewChange)). @@ -474,6 +490,8 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.mutex.Lock() defer consensus.mutex.Unlock() + fmt.Printf("[onNewView] received new view message from %+v\n", recvMsg) + consensus.getLogger().Info(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -562,6 +580,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // newView message verified success, override my state consensus.SetViewIDs(recvMsg.ViewID) consensus.pubKeyLock.Lock() + fmt.Println("[onNewView1221] new leader key cur:", consensus.LeaderPubKey.Bytes.Hex(), " new: ", senderKey.Bytes.Hex()) consensus.LeaderPubKey = senderKey consensus.pubKeyLock.Unlock() consensus.ResetViewChangeState() diff --git a/hmy/hmy.go b/hmy/hmy.go index 58e8d59edf..8089f04ba8 100644 --- a/hmy/hmy.go +++ b/hmy/hmy.go @@ -18,6 +18,7 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + "github.com/harmony-one/harmony/multibls" commonRPC "github.com/harmony-one/harmony/rpc/common" "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" @@ -96,6 +97,7 @@ type NodeAPI interface { GetStakingTransactionsCount(address, txType string) (uint64, error) GetTraceResultByHash(hash common.Hash) (json.RawMessage, error) IsCurrentlyLeader() bool + GetPublicKeys() multibls.PublicKeys IsOutOfSync(shardID uint32) bool SyncStatus(shardID uint32) (bool, uint64, uint64) SyncPeers() map[string]int diff --git a/node/api.go b/node/api.go index ceda968084..debcb201f6 100644 --- a/node/api.go +++ b/node/api.go @@ -6,6 +6,7 @@ import ( "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy" "github.com/harmony-one/harmony/internal/tikv" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/rosetta" hmy_rpc "github.com/harmony-one/harmony/rpc" rpc_common "github.com/harmony-one/harmony/rpc/common" @@ -18,6 +19,11 @@ func (node *Node) IsCurrentlyLeader() bool { return node.Consensus.IsLeader() } +// GetPublicKeys exposes if node is currently the leader node +func (node *Node) GetPublicKeys() multibls.PublicKeys { + return node.Consensus.GetPrivateKeys().GetPublicKeys() +} + // PeerConnectivity .. func (node *Node) PeerConnectivity() (int, int, int) { return node.host.PeerConnectivity() diff --git a/node/node_newblock.go b/node/node_newblock.go index 03fd69d9dd..24e26ad99a 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -2,6 +2,7 @@ package node import ( "errors" + "fmt" "sort" "strings" "time" @@ -88,6 +89,8 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp newBlock, err := node.ProposeNewBlock(newCommitSigsChan) if err == nil { + fmt.Printf("ProposeNewBlock: #%d :%d @%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), newBlock.Header().ViewID().Int64(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) + if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). Msg("Block with the same number was already proposed, abort.") @@ -145,6 +148,7 @@ func (node *Node) ProposeNewBlock(commitSigs chan []byte) (*types.Block, error) if node.Blockchain().Config().IsStaking(header.Epoch()) { blsPubKeyBytes := leaderKey.Object.GetAddress() coinbase.SetBytes(blsPubKeyBytes[:]) + fmt.Println("coinbase.SetBytes leader: ", leaderKey.Bytes.Hex(), coinbase.Hex()) } emptyAddr := common.Address{} diff --git a/test/configs/local-resharding.txt b/test/configs/local-resharding.txt index 4ce91ece1c..fc3d3e97b4 100644 --- a/test/configs/local-resharding.txt +++ b/test/configs/local-resharding.txt @@ -1,25 +1,12 @@ 127.0.0.1 9000 validator .hmy/65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204.key -127.0.0.1 9002 validator .hmy/40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809.key -127.0.0.1 9004 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key -127.0.0.1 9006 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key -127.0.0.1 9008 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key -127.0.0.1 9010 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key -127.0.0.1 9012 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key -127.0.0.1 9014 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key -127.0.0.1 9016 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key -127.0.0.1 9018 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key -127.0.0.1 9020 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key -127.0.0.1 9022 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key -127.0.0.1 9200 explorer null 0 +127.0.0.1 9002 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key +127.0.0.1 9004 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key +127.0.0.1 9006 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key +127.0.0.1 9008 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key +127.0.0.1 9010 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key +127.0.0.1 9099 explorer null 0 127.0.0.1 9100 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key -127.0.0.1 9102 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key -127.0.0.1 9104 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key -127.0.0.1 9106 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key -127.0.0.1 9108 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key -127.0.0.1 9110 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key -127.0.0.1 9112 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key -127.0.0.1 9114 validator .hmy/f47238daef97d60deedbde5302d05dea5de67608f11f406576e363661f7dcbc4a1385948549b31a6c70f6fde8a391486.key -127.0.0.1 9116 validator .hmy/fc4b9c535ee91f015efff3f32fbb9d32cdd9bfc8a837bb3eee89b8fff653c7af2050a4e147ebe5c7233dc2d5df06ee0a.key -127.0.0.1 9118 validator .hmy/ca86e551ee42adaaa6477322d7db869d3e203c00d7b86c82ebee629ad79cb6d57b8f3db28336778ec2180e56a8e07296.key -127.0.0.1 9300 explorer null 1 \ No newline at end of file +127.0.0.1 9102 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key +127.0.0.1 9104 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key +127.0.0.1 9106 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key \ No newline at end of file diff --git a/test/deploy.sh b/test/deploy.sh index 9ab6943110..fe22de57f0 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -69,7 +69,7 @@ function launch_localnet() { if ${VERBOSE}; then verbosity=5 else - verbosity=3 + verbosity=5 fi base_args=(--log_folder "${log_folder}" --min_peers "${MIN}" --bootnodes "${BN_MA}" "--network_type=$NETWORK" --blspass file:"${ROOT}/.hmy/blspass.txt" "--dns=false" "--verbosity=${verbosity}" "--p2p.security.max-conn-per-ip=100") @@ -80,8 +80,11 @@ function launch_localnet() { while IFS='' read -r line || [[ -n "$line" ]]; do i=$((i + 1)) + + # Read config for i-th node form config file IFS=' ' read -r ip port mode bls_key shard node_config <<<"${line}" + echo "LINE: ${line} ${shard}" args=("${base_args[@]}" --ip "${ip}" --port "${port}" --key "/tmp/${ip}-${port}.key" --db_dir "${ROOT}/db-${ip}-${port}" "--broadcast_invalid_tx=false") if [[ -z "$ip" || -z "$port" ]]; then echo "skip empty node" From 6283a319ed4ffc33748e36ff50348b4648cde6aa Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 5 Sep 2022 19:39:49 +0700 Subject: [PATCH 062/420] consensus check is forked --- api/service/explorer/service.go | 6 ++-- consensus/consensus.go | 2 -- consensus/consensus_service.go | 14 -------- consensus/consensus_v2.go | 61 ++++++++++++++++++++++++--------- internal/params/config.go | 11 ++++++ 5 files changed, 58 insertions(+), 36 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index d96c1bc56c..5887bb3a02 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -223,9 +223,9 @@ func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { for i := cur; i > 0; i-- { block := s.blockchain.GetBlockByNumber(i) - w.Write([]byte(fmt.Sprintf("%d ", i))) - w.Write([]byte(fmt.Sprintf("%s ", block.Header().ViewID().String()))) - w.Write([]byte(fmt.Sprintf("%s ", block.Header().Coinbase().Hash().Hex()))) + w.Write([]byte(fmt.Sprintf("#%d ", i))) + w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) + w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) } } diff --git a/consensus/consensus.go b/consensus/consensus.go index 0c51d648c0..e0e096e306 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -73,8 +73,6 @@ type Consensus struct { priKey multibls.PrivateKeys // the publickey of leader LeaderPubKey *bls.PublicKeyWrapper - // index of leader in the list of validators. - LeaderIndex int // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e646504dde..d26ea8ced5 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -474,20 +474,6 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } -// SetLeaderIndex set the leader index. -func (consensus *Consensus) SetLeaderIndex(f func(int) int) (current int) { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() - consensus.LeaderIndex = f(consensus.LeaderIndex) - return consensus.LeaderIndex -} - -func (consensus *Consensus) GetLeaderIndex() int { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() - return consensus.LeaderIndex -} - // SetViewChangingID set the current view change ID func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index bf042c62d3..d92a417836 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -737,25 +737,41 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) // first view id is going to be 2. prev := consensus.GetLeaderPubKey() - idx := consensus.SetLeaderIndex(func(i int) int { - if curBlockViewID%3 == 0 { - return i + 1 + if consensus.Blockchain.Config().IsLeaderRotation(blk.Epoch()) { + epochBlockViewID, err := consensus.getEpochFirstBlockViewID(blk.Epoch()) + if err != nil { + consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", blk.Epoch().Uint64()) + return + } + if epochBlockViewID > curBlockViewID { + consensus.getLogger().Error().Msg("[SetupForNewConsensus] Epoch block viewID is greater than current block viewID") + return } - return i - }) - pps := consensus.Decider.Participants() - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = &pps[idx%len(pps)] - fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %q\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) - consensus.pubKeyLock.Unlock() - if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { - // leader changed - go func() { - fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - consensus.ReadySignal <- SyncProposal - }() + diff := curBlockViewID - epochBlockViewID + + pps := consensus.Decider.Participants() + idx := (int(diff) / 3) % len(pps) + consensus.pubKeyLock.Lock() + fmt.Println("(int(diff)/3)%len(pps) == ", idx) + consensus.LeaderPubKey = &pps[idx] + fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + consensus.pubKeyLock.Unlock() + if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { + // leader changed + go func() { + fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + consensus.ReadySignal <- SyncProposal + + }() + } + } else { + fmt.Printf("SetupForNewConsensus0 :%d future v%d new: %s prev: %s %v\n", utils.GetPort(), curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] + consensus.pubKeyLock.Unlock() + } } var epoch *big.Int @@ -776,6 +792,17 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.ResetState() } +func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, error) { + if epoch.Uint64() == 0 { + return 0, nil + } + epochBlock := consensus.Blockchain.GetBlockByNumber(epoch.Uint64() - 1) + if epochBlock == nil { + return 0, errors.Errorf("block not found for number %d", epoch.Uint64()-1) + } + return epochBlock.Header().ViewID().Uint64() + 1, nil +} + func (consensus *Consensus) postCatchup(initBN uint64) { if initBN < consensus.BlockNum() { consensus.getLogger().Info(). diff --git a/internal/params/config.go b/internal/params/config.go index c11daeafb5..29af19c74f 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -271,6 +271,8 @@ var ( SlotsLimitedEpoch: EpochTBD, // epoch to enable HIP-16 CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, + LeaderRotationEpoch: big.NewInt(1), + TesnetNinetyPercentEpoch: EpochTBD, FeeCollectEpoch: big.NewInt(5), LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 5, @@ -315,6 +317,7 @@ var ( big.NewInt(0), // AllowlistEpoch big.NewInt(1), // LeaderRotationEpoch 64, // LeaderRotationBlocksCount + big.NewInt(1), // LeaderRotationEpoch big.NewInt(0), // FeeCollectEpoch } @@ -711,6 +714,14 @@ func (c *ChainConfig) IsLeaderRotation(epoch *big.Int) bool { return isForked(c.LeaderRotationEpoch, epoch) } +func (c *ChainConfig) IsLeaderRotation(epoch *big.Int) bool { + return isForked(c.LeaderRotationEpoch, epoch) +} + +func (c *ChainConfig) IsTestnetNinetyPercent(epoch *big.Int) bool { + return isForked(c.TesnetNinetyPercentEpoch, epoch) && c == TestnetChainConfig +} + // IsFeeCollectEpoch determines whether Txn Fees will be collected into the community-managed account. func (c *ChainConfig) IsFeeCollectEpoch(epoch *big.Int) bool { return isForked(c.FeeCollectEpoch, epoch) From af3a293bfed8502dfe60191db2e2f1699ffc6d83 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sat, 10 Sep 2022 18:02:45 +0700 Subject: [PATCH 063/420] fix --- api/service/explorer/service.go | 2 -- cmd/harmony/main.go | 3 --- consensus/consensus_service.go | 11 ----------- consensus/quorum/quorum.go | 7 +------ 4 files changed, 1 insertion(+), 22 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 5887bb3a02..6aa275332b 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -115,8 +115,6 @@ func (s *Service) Run() *http.Server { s.router = mux.NewRouter() - fmt.Println("++", addr) - // Set up router for addresses. // Fetch addresses request, accepts parameter size: how much addresses to read, // parameter prefix: from which address prefix start diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 2fcfe5f037..4bb70a81b4 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -266,8 +266,6 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { var err error - fmt.Println("OS: ", os.Args) - nodeconfigSetShardSchedule(hc) nodeconfig.SetShardingSchedule(shard.Schedule) nodeconfig.SetVersion(getHarmonyVersion()) @@ -803,7 +801,6 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // Set the consensus ID to be the current block number viewID := currentNode.Blockchain().CurrentBlock().Header().ViewID().Uint64() - fmt.Println("viewID:", viewID) currentConsensus.SetViewIDs(viewID + 1) utils.Logger().Info(). Uint64("viewID", viewID). diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index d26ea8ced5..b010251217 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -213,7 +213,6 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } - fmt.Println("[checkViewID] Set LEADEER PUB KEY ", msg.SenderPubkeys[0].Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() @@ -397,7 +396,6 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { Str("leaderPubKey", leaderPubKey.Bytes.Hex()). Msg("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain") consensus.pubKeyLock.Lock() - fmt.Println("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain", leaderPubKey.Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = leaderPubKey consensus.pubKeyLock.Unlock() } @@ -484,15 +482,6 @@ func (consensus *Consensus) StartFinalityCount() { consensus.finalityCounter.Store(time.Now().UnixNano()) } -//func (consensus *Consensus) ReshardingNextLeader(newblock *types.Block) { -// consensus.pubKeyLock.Lock() -// fmt.Println("nextBlock1 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) -// consensus.LeaderPubKey = consensus.getNextLeaderKey(consensus.GetCurBlockViewID() + 1) -// fmt.Println("nextBlock2 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) -// consensus.pubKeyLock.Unlock() -// -//} - // FinishFinalityCount calculate the current finality func (consensus *Consensus) FinishFinalityCount() { d := time.Now().UnixNano() diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 0586ace4fe..6cf79e549a 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -230,17 +230,12 @@ func (s *cIdentities) NthNextHmy(instance shardingconfig.Instance, pubKey *bls.P Msg("[NthNextHmy] pubKey not found") } numNodes := instance.NumHarmonyOperatedNodesPerShard() - //fmt.Println("??idx:", idx, numNodes) // sanity check to avoid out of bound access if numNodes <= 0 || numNodes > len(s.publicKeys) { numNodes = len(s.publicKeys) } idx = (idx + next) % numNodes - //fmt.Println("-------idx:", idx) - - new := &s.publicKeys[idx] - fmt.Println("NthNextHmy: ", pubKey.Bytes.Hex(), new.Bytes.Hex()) - return found, new + return found, &s.publicKeys[idx] } // NthNextHmyExt return the Nth next pubkey of Harmony + allowlist nodes, next can be negative number From 2d95758cb699e45e8545bc3948f8525009626b74 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 15:27:56 +0700 Subject: [PATCH 064/420] Cleanup and fix update pub keys. --- cmd/harmony/main.go | 10 ++++++++++ consensus/consensus.go | 2 +- consensus/consensus_service.go | 4 ++++ consensus/consensus_v2.go | 1 - consensus/leader.go | 11 +++-------- consensus/view_change.go | 11 ----------- 6 files changed, 18 insertions(+), 21 deletions(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 4bb70a81b4..cea2be6c25 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -677,6 +677,16 @@ func createGlobalConfig(hc harmonyconfig.HarmonyConfig) (*nodeconfig.ConfigType, } func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfig.ConfigType, registry *registry.Registry) *node.Node { + // Consensus object. + // TODO: consensus object shouldn't start here + decider := quorum.NewDecider(quorum.SuperMajorityVote, uint32(hc.General.ShardID)) + currentConsensus, err := consensus.New( + myHost, nodeConfig.ShardID, p2p.Peer{}, nodeConfig.ConsensusPriKey, decider) + + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Error :%v \n", err) + os.Exit(1) + } // Parse minPeers from harmonyconfig.HarmonyConfig var minPeers int var aggregateSig bool diff --git a/consensus/consensus.go b/consensus/consensus.go index e0e096e306..ec9d79ce14 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -230,7 +230,7 @@ func (consensus *Consensus) BlockNum() uint64 { // New create a new Consensus record func New( - host p2p.Host, shard uint32, multiBLSPriKey multibls.PrivateKeys, + host p2p.Host, shard uint32, leader p2p.Peer, multiBLSPriKey multibls.PrivateKeys, registry *registry.Registry, Decider quorum.Decider, minPeers int, aggregateSig bool, ) (*Consensus, error) { diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index b010251217..5055684e6c 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -74,6 +74,10 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { + if utils.GetPort() == 9000 { + //utils.Logger().Info().Msg("UpdatePublicKeys") + fmt.Println("UpdatePublicKeys", len(pubKeys), len(allowlist)) + } // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index d92a417836..6b2d6f7025 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -650,7 +650,6 @@ func (consensus *Consensus) tryCatchup() error { consensus.getLogger().Error().Err(err).Msg("[TryCatchup] Failed to add block to chain") return err } - //fmt.Println("tryCatchup ", utils.GetPort(), blk.NumberU64()) select { // TODO: Remove this when removing dns sync and stream sync is fully up case consensus.VerifiedNewBlock <- blk: diff --git a/consensus/leader.go b/consensus/leader.go index a359c229a4..4f72ae6268 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -200,16 +200,12 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { // TODO HERE - //if recvMsg.ViewID == 10 { - // return - //} + if recvMsg.ViewID == 21 { + return + } consensus.mutex.Lock() defer consensus.mutex.Unlock() //// Read - Start - if consensus.ShardID == 0 { - //fmt.Println("onCommit ", recvMsg.BlockNum) - } - if !consensus.isRightBlockNumAndViewID(recvMsg) { return } @@ -341,5 +337,4 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED) } - //fmt.Println("onCommit99: ", utils.GetPort(), recvMsg.BlockNum) } diff --git a/consensus/view_change.go b/consensus/view_change.go index fd9cde6c5f..3bd0c181f2 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync" "time" @@ -162,7 +161,6 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { Uint64("stuckBlockViewID", stuckBlockViewID). Msg("[getNextViewID]") - fmt.Println("end getNextViewID: ", nextViewID, viewChangeDuration) // duration is always the fixed view change duration for synchronous view change return nextViewID, viewChangeDuration } @@ -235,7 +233,6 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe lastLeaderPubKey, gap) } - fmt.Println("wasfoundNext", consensus.Blockchain.Config().IsAllowlistEpoch(epoch), wasFound, next.Bytes.Hex(), lastLeaderPubKey.Bytes.Hex()) if !wasFound { consensus.getLogger().Warn(). Str("key", consensus.LeaderPubKey.Bytes.Hex()). @@ -257,7 +254,6 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - fmt.Printf("Message to send leader111: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } @@ -268,7 +264,6 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() - //fmt.Println("startViewChange", nextViewID) consensus.SetViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet @@ -312,9 +307,7 @@ func (consensus *Consensus) startViewChange() { if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { continue } - // Тут уже другой leader msgToSend := consensus.constructViewChangeMessage(&key) - fmt.Println("Message to send leader222: ", consensus.LeaderPubKey.Bytes.Hex()) if err := consensus.msgSender.SendWithRetry( consensus.BlockNum(), msg_pb.MessageType_VIEWCHANGE, @@ -372,7 +365,6 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.ResetState() } - fmt.Println("[startNewView]", newLeaderPriKey.Pub.Bytes.Hex()) consensus.SetLeaderPubKey(newLeaderPriKey.Pub) return nil @@ -490,8 +482,6 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.mutex.Lock() defer consensus.mutex.Unlock() - fmt.Printf("[onNewView] received new view message from %+v\n", recvMsg) - consensus.getLogger().Info(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -580,7 +570,6 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // newView message verified success, override my state consensus.SetViewIDs(recvMsg.ViewID) consensus.pubKeyLock.Lock() - fmt.Println("[onNewView1221] new leader key cur:", consensus.LeaderPubKey.Bytes.Hex(), " new: ", senderKey.Bytes.Hex()) consensus.LeaderPubKey = senderKey consensus.pubKeyLock.Unlock() consensus.ResetViewChangeState() From 0880e38ab03d184732e76d02b87468086ed3fa4c Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 13 Sep 2022 01:10:13 +0700 Subject: [PATCH 065/420] fix fix fix fix fix --- consensus/consensus_service.go | 5 ----- consensus/leader.go | 4 ---- internal/params/config.go | 9 ++++++--- node/node_newblock.go | 2 +- 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 5055684e6c..1844f22687 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync/atomic" "time" @@ -74,10 +73,6 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { - if utils.GetPort() == 9000 { - //utils.Logger().Info().Msg("UpdatePublicKeys") - fmt.Println("UpdatePublicKeys", len(pubKeys), len(allowlist)) - } // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) diff --git a/consensus/leader.go b/consensus/leader.go index 4f72ae6268..422508762b 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -199,10 +199,6 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { - // TODO HERE - if recvMsg.ViewID == 21 { - return - } consensus.mutex.Lock() defer consensus.mutex.Unlock() //// Read - Start diff --git a/internal/params/config.go b/internal/params/config.go index 29af19c74f..3d8f9762f8 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -271,7 +271,8 @@ var ( SlotsLimitedEpoch: EpochTBD, // epoch to enable HIP-16 CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, - LeaderRotationEpoch: big.NewInt(1), + LeaderRotationEpoch: big.NewInt(4), + LeaderRotationBlocksCount: 5, TesnetNinetyPercentEpoch: EpochTBD, FeeCollectEpoch: big.NewInt(5), LeaderRotationEpoch: EpochTBD, @@ -318,6 +319,7 @@ var ( big.NewInt(1), // LeaderRotationEpoch 64, // LeaderRotationBlocksCount big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } @@ -358,8 +360,9 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount + + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } diff --git a/node/node_newblock.go b/node/node_newblock.go index 24e26ad99a..d59a563ebc 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -89,7 +89,7 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp newBlock, err := node.ProposeNewBlock(newCommitSigsChan) if err == nil { - fmt.Printf("ProposeNewBlock: #%d :%d @%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), newBlock.Header().ViewID().Int64(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) + fmt.Printf("ProposeNewBlock: #%d :%d e:%d, sh:%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), node.Blockchain().CurrentHeader().Epoch().Uint64(), node.Blockchain().ShardID(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). From 866f2b02bf741a6e4fbdb4c6545bc76268b45499 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 17 Oct 2022 15:08:07 +0700 Subject: [PATCH 066/420] activate epoch --- internal/params/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/params/config.go b/internal/params/config.go index 3d8f9762f8..f1000098a3 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -111,6 +111,7 @@ var ( AllowlistEpoch: big.NewInt(2), LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 64, + TesnetNinetyPercentEpoch: big.NewInt(399), FeeCollectEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. From 71680e8e80271d098dd1df975fa4729506b74bce Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 17 Oct 2022 17:09:04 +0700 Subject: [PATCH 067/420] EpochTBD for leader rotation epoch. --- internal/params/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/params/config.go b/internal/params/config.go index f1000098a3..fc61123aa8 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -110,7 +110,7 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(2), AllowlistEpoch: big.NewInt(2), LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 64, + LeaderRotationBlocksCount: 64, TesnetNinetyPercentEpoch: big.NewInt(399), FeeCollectEpoch: EpochTBD, } From a0b1d9ea5c72512f7b4ff1dcac765aa5242085c1 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 18 Oct 2022 12:17:17 +0700 Subject: [PATCH 068/420] 295 epoch --- internal/params/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/params/config.go b/internal/params/config.go index fc61123aa8..f1000098a3 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -110,7 +110,7 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(2), AllowlistEpoch: big.NewInt(2), LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 64, + LeaderRotationBlocksCount: 64, TesnetNinetyPercentEpoch: big.NewInt(399), FeeCollectEpoch: EpochTBD, } From 471042773bbaf6b167c0d5ff42e1690211e15e06 Mon Sep 17 00:00:00 2001 From: Konstantin <355847+Frozen@users.noreply.github.com> Date: Fri, 28 Oct 2022 12:52:36 +0300 Subject: [PATCH 069/420] Decider no longer requires public keys as a dependency. (#4289) --- cmd/harmony/main.go | 2 +- consensus/consensus.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index cea2be6c25..d89254ead7 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -681,7 +681,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // TODO: consensus object shouldn't start here decider := quorum.NewDecider(quorum.SuperMajorityVote, uint32(hc.General.ShardID)) currentConsensus, err := consensus.New( - myHost, nodeConfig.ShardID, p2p.Peer{}, nodeConfig.ConsensusPriKey, decider) + myHost, nodeConfig.ShardID, nodeConfig.ConsensusPriKey, decider) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "Error :%v \n", err) diff --git a/consensus/consensus.go b/consensus/consensus.go index ec9d79ce14..e0e096e306 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -230,7 +230,7 @@ func (consensus *Consensus) BlockNum() uint64 { // New create a new Consensus record func New( - host p2p.Host, shard uint32, leader p2p.Peer, multiBLSPriKey multibls.PrivateKeys, + host p2p.Host, shard uint32, multiBLSPriKey multibls.PrivateKeys, registry *registry.Registry, Decider quorum.Decider, minPeers int, aggregateSig bool, ) (*Consensus, error) { From aa98ca914828f580c10d3e31480be734042a541f Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 19 Oct 2022 22:26:13 +0700 Subject: [PATCH 070/420] Consensus doesn't require anymore `Node` as a circular dependency. --- core/blockchain_impl.go | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 8b0683bd65..783a571e89 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -327,19 +327,8 @@ func VerifyBlockCrossLinks(blockchain BlockChain, block *types.Block) error { // ReadCrossLink beacon chain usage. cl, err := blockchain.ReadCrossLink(crossLink.ShardID(), crossLink.BlockNum()) if err == nil && cl != nil { - utils.Logger().Err(errAlreadyExist). - Uint64("beacon-block-number", block.NumberU64()). - Interface("remote", crossLink). - Interface("local", cl). - Msg("[CrossLinkVerification]") - // TODO Add slash for exist same blocknum but different crosslink - return errors.Wrapf( - errAlreadyExist, - "[CrossLinkVerification] shard: %d block: %d on beacon block %d", - crossLink.ShardID(), - crossLink.BlockNum(), - block.NumberU64(), - ) + // Add slash for exist same blocknum but different crosslink + return errAlreadyExist } if err := VerifyCrossLink(blockchain, crossLink); err != nil { return errors.Wrapf(err, "cannot VerifyBlockCrossLinks") From 1aefca345dc55a7aa22114b950d10a3294b45cfc Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Fri, 28 Oct 2022 00:48:12 +0700 Subject: [PATCH 071/420] Proper blockchain initialization. --- cmd/harmony/main.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index d89254ead7..4bb70a81b4 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -677,16 +677,6 @@ func createGlobalConfig(hc harmonyconfig.HarmonyConfig) (*nodeconfig.ConfigType, } func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfig.ConfigType, registry *registry.Registry) *node.Node { - // Consensus object. - // TODO: consensus object shouldn't start here - decider := quorum.NewDecider(quorum.SuperMajorityVote, uint32(hc.General.ShardID)) - currentConsensus, err := consensus.New( - myHost, nodeConfig.ShardID, nodeConfig.ConsensusPriKey, decider) - - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "Error :%v \n", err) - os.Exit(1) - } // Parse minPeers from harmonyconfig.HarmonyConfig var minPeers int var aggregateSig bool From a92cc1bb409c9344303c0afc0b67daf7ae6b156f Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 00:33:16 +0800 Subject: [PATCH 072/420] Rwlock consensus. --- api/service/consensus/service.go | 18 ++-- consensus/consensus.go | 17 ++-- consensus/consensus_test.go | 6 +- consensus/consensus_v2.go | 146 +++++++++++++++++-------------- node/node.go | 2 - node/node_newblock.go | 2 +- node/service_setup.go | 2 +- 7 files changed, 99 insertions(+), 94 deletions(-) diff --git a/api/service/consensus/service.go b/api/service/consensus/service.go index d48c660a76..e31ae71bbe 100644 --- a/api/service/consensus/service.go +++ b/api/service/consensus/service.go @@ -3,23 +3,21 @@ package consensus import ( msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus" - "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/utils" ) // Service is the consensus service. type Service struct { - blockChannel chan *types.Block // The channel to receive new blocks from Node - consensus *consensus.Consensus - stopChan chan struct{} - stoppedChan chan struct{} - startChan chan struct{} - messageChan chan *msg_pb.Message + consensus *consensus.Consensus + stopChan chan struct{} + stoppedChan chan struct{} + startChan chan struct{} + messageChan chan *msg_pb.Message } // New returns consensus service. -func New(blockChannel chan *types.Block, consensus *consensus.Consensus, startChan chan struct{}) *Service { - return &Service{blockChannel: blockChannel, consensus: consensus, startChan: startChan} +func New(consensus *consensus.Consensus, startChan chan struct{}) *Service { + return &Service{consensus: consensus, startChan: startChan} } // Start starts consensus service. @@ -27,7 +25,7 @@ func (s *Service) Start() error { utils.Logger().Info().Msg("[consensus/service] Starting consensus service.") s.stopChan = make(chan struct{}) s.stoppedChan = make(chan struct{}) - s.consensus.Start(s.blockChannel, s.stopChan, s.stoppedChan, s.startChan) + s.consensus.Start(s.stopChan, s.stoppedChan, s.startChan) s.consensus.WaitForNewRandomness() return nil } diff --git a/consensus/consensus.go b/consensus/consensus.go index e0e096e306..dbfb39720e 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -85,7 +85,7 @@ type Consensus struct { // IgnoreViewIDCheck determines whether to ignore viewID check IgnoreViewIDCheck *abool.AtomicBool // consensus mutex - mutex sync.Mutex + mutex sync.RWMutex // mutex for verify new block verifyBlockMutex sync.Mutex // ViewChange struct @@ -114,10 +114,6 @@ type Consensus struct { host p2p.Host // MessageSender takes are of sending consensus message and the corresponding retry logic. msgSender *MessageSender - // Used to convey to the consensus main loop that block syncing has finished. - syncReadyChan chan struct{} - // Used to convey to the consensus main loop that node is out of sync - syncNotReadyChan chan struct{} // If true, this consensus will not propose view change. disableViewChange bool // Have a dedicated reader thread pull from this chan, like in node @@ -136,6 +132,9 @@ type Consensus struct { finalityCounter atomic.Value //int64 dHelper *downloadHelper + + // Flag only for initialization state. + start bool } // Blockchain returns the blockchain. @@ -157,12 +156,14 @@ func (consensus *Consensus) VerifyBlock(block *types.Block) error { // BlocksSynchronized lets the main loop know that block synchronization finished // thus the blockchain is likely to be up to date. func (consensus *Consensus) BlocksSynchronized() { - consensus.syncReadyChan <- struct{}{} + consensus.mutex.Lock() + consensus.syncReadyChan() + consensus.mutex.Unlock() } // BlocksNotSynchronized lets the main loop know that block is not synchronized func (consensus *Consensus) BlocksNotSynchronized() { - consensus.syncNotReadyChan <- struct{}{} + consensus.syncNotReadyChan() } // VdfSeedSize returns the number of VRFs for VDF computation @@ -265,8 +266,6 @@ func New( // displayed on explorer as Height right now consensus.SetCurBlockViewID(0) consensus.ShardID = shard - consensus.syncReadyChan = make(chan struct{}) - consensus.syncNotReadyChan = make(chan struct{}) consensus.SlashChan = make(chan slash.Record) consensus.ReadySignal = make(chan ProposalType) consensus.CommitSigChannel = make(chan []byte) diff --git a/consensus/consensus_test.go b/consensus/consensus_test.go index c2d3d729b6..7d74d8815a 100644 --- a/consensus/consensus_test.go +++ b/consensus/consensus_test.go @@ -61,11 +61,7 @@ func TestConsensusInitialization(t *testing.T) { assert.Equal(t, uint64(0), consensus.GetViewChangingID()) assert.Equal(t, uint32(shard.BeaconChainShardID), consensus.ShardID) - assert.IsType(t, make(chan struct{}), consensus.syncReadyChan) - assert.NotNil(t, consensus.syncReadyChan) - - assert.IsType(t, make(chan struct{}), consensus.syncNotReadyChan) - assert.NotNil(t, consensus.syncNotReadyChan) + assert.Equal(t, false, consensus.start) assert.IsType(t, make(chan slash.Record), consensus.SlashChan) assert.NotNil(t, consensus.SlashChan) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 6b2d6f7025..d849898d60 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -292,7 +292,7 @@ func (consensus *Consensus) BlockCommitSigs(blockNum uint64) ([]byte, error) { // Start waits for the next new block and run consensus func (consensus *Consensus) Start( - blockChannel chan *types.Block, stopChan, stoppedChan, startChannel chan struct{}, + stopChan, stoppedChan, startChannel chan struct{}, ) { go func() { toStart := make(chan struct{}, 1) @@ -317,13 +317,13 @@ func (consensus *Consensus) Start( // Set up next block due time. consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) - start := false + consensus.start = false for { select { case <-toStart: - start = true + consensus.start = true case <-ticker.C: - if !start && isInitialLeader { + if !consensus.start && isInitialLeader { continue } for k, v := range consensus.consensusTimeout { @@ -362,68 +362,6 @@ func (consensus *Consensus) Start( } } - // TODO: Refactor this piece of code to consensus/downloader.go after DNS legacy sync is removed - case <-consensus.syncReadyChan: - consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncReadyChan") - consensus.mutex.Lock() - if consensus.BlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { - consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) - consensus.SetViewIDs(consensus.Blockchain().CurrentHeader().ViewID().Uint64() + 1) - mode := consensus.UpdateConsensusInformation() - consensus.current.SetMode(mode) - consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") - consensus.consensusTimeout[timeoutConsensus].Start() - consensus.getLogger().Info().Str("Mode", mode.String()).Msg("Node is IN SYNC") - consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() - } else if consensus.Mode() == Syncing { - // Corner case where sync is triggered before `onCommitted` and there is a race - // for block insertion between consensus and downloader. - mode := consensus.UpdateConsensusInformation() - consensus.SetMode(mode) - consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") - consensus.consensusTimeout[timeoutConsensus].Start() - consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() - } - consensus.mutex.Unlock() - - // TODO: Refactor this piece of code to consensus/downloader.go after DNS legacy sync is removed - case <-consensus.syncNotReadyChan: - consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncNotReadyChan") - consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) - consensus.current.SetMode(Syncing) - consensus.getLogger().Info().Msg("[ConsensusMainLoop] Node is OUT OF SYNC") - consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() - - case newBlock := <-blockChannel: - //consensus.ReshardingNextLeader(newBlock) - consensus.getLogger().Info(). - Uint64("MsgBlockNum", newBlock.NumberU64()). - Msg("[ConsensusMainLoop] Received Proposed New Block!") - - if newBlock.NumberU64() < consensus.BlockNum() { - consensus.getLogger().Warn().Uint64("newBlockNum", newBlock.NumberU64()). - Msg("[ConsensusMainLoop] received old block, abort") - continue - } - // Sleep to wait for the full block time - consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") - - <-time.After(time.Until(consensus.NextBlockDue)) - consensus.StartFinalityCount() - - // Update time due for next block - consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) - - startTime = time.Now() - consensus.msgSender.Reset(newBlock.NumberU64()) - - consensus.getLogger().Info(). - Int("numTxs", len(newBlock.Transactions())). - Int("numStakingTxs", len(newBlock.StakingTransactions())). - Time("startTime", startTime). - Int64("publicKeys", consensus.Decider.ParticipantsCount()). - Msg("[ConsensusMainLoop] STARTING CONSENSUS") - consensus.announce(newBlock) case <-stopChan: consensus.getLogger().Info().Msg("[ConsensusMainLoop] stopChan") return @@ -436,6 +374,82 @@ func (consensus *Consensus) Start( } } +func (consensus *Consensus) syncReadyChan() { + consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncReadyChan") + if consensus.BlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { + consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) + consensus.SetViewIDs(consensus.Blockchain().CurrentHeader().ViewID().Uint64() + 1) + mode := consensus.UpdateConsensusInformation() + consensus.current.SetMode(mode) + consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") + consensus.consensusTimeout[timeoutConsensus].Start() + consensus.getLogger().Info().Str("Mode", mode.String()).Msg("Node is IN SYNC") + consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() + } else if consensus.Mode() == Syncing { + // Corner case where sync is triggered before `onCommitted` and there is a race + // for block insertion between consensus and downloader. + mode := consensus.UpdateConsensusInformation() + consensus.SetMode(mode) + consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") + consensus.consensusTimeout[timeoutConsensus].Start() + consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() + } + +} + +func (consensus *Consensus) syncNotReadyChan() { + consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncNotReadyChan") + consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) + consensus.current.SetMode(Syncing) + consensus.getLogger().Info().Msg("[ConsensusMainLoop] Node is OUT OF SYNC") + consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() +} + +// Close close the consensus. If current is in normal commit phase, wait until the commit +// phase end. +func (consensus *Consensus) Close() error { + if consensus.dHelper != nil { + consensus.dHelper.close() + } + consensus.waitForCommit() + return nil +} + +func (consensus *Consensus) BlockChannel(newBlock *types.Block) { + //consensus.ReshardingNextLeader(newBlock)consensus.getLogger().Info(). + Uint64("MsgBlockNum", newBlock.NumberU64()). + Msg("[ConsensusMainLoop] Received Proposed New Block!") + + if newBlock.NumberU64() < consensus.BlockNum() { + consensus.getLogger().Warn().Uint64("newBlockNum", newBlock.NumberU64()). + Msg("[ConsensusMainLoop] received old block, abort") + return + } + // Sleep to wait for the full block time + consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") + time.AfterFunc(time.Until(consensus.NextBlockDue), func() { + consensus.StartFinalityCount() + + // Update time due for next block + consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) + + startTime = time.Now() + consensus.msgSender.Reset(newBlock.NumberU64()) + + consensus.getLogger().Info(). + Int("numTxs", len(newBlock.Transactions())). + Int("numStakingTxs", len(newBlock.StakingTransactions())). + Time("startTime", startTime). + Int64("publicKeys", consensus.Decider.ParticipantsCount()). + Msg("[ConsensusMainLoop] STARTING CONSENSUS") + consensus.announce(newBlock) + }) + + if consensus.dHelper != nil { + consensus.dHelper.start() + } +} + // Close closes the consensus. If current is in normal commit phase, wait until the commit // phase end. func (consensus *Consensus) Close() error { diff --git a/node/node.go b/node/node.go index 6c2ebcd5fe..a5c1288876 100644 --- a/node/node.go +++ b/node/node.go @@ -100,7 +100,6 @@ type ISync interface { // Node represents a protocol-participating node in the network type Node struct { Consensus *consensus.Consensus // Consensus object containing all Consensus related data (e.g. committee members, signatures, commits) - BlockChannel chan *types.Block // The channel to send newly proposed blocks ConfirmedBlockChannel chan *types.Block // The channel to send confirmed blocks BeaconBlockChannel chan *types.Block // The channel to send beacon blocks for non-beaconchain nodes pendingCXReceipts map[string]*types.CXReceiptsProof // All the receipts received but not yet processed for Consensus @@ -1083,7 +1082,6 @@ func New( } } - node.BlockChannel = make(chan *types.Block) node.ConfirmedBlockChannel = make(chan *types.Block) node.BeaconBlockChannel = make(chan *types.Block) txPoolConfig := core.DefaultTxPoolConfig diff --git a/node/node_newblock.go b/node/node_newblock.go index d59a563ebc..c29fb1b96a 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -107,7 +107,7 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp // Send the new block to Consensus so it can be confirmed. node.proposedBlock[newBlock.NumberU64()] = newBlock delete(node.proposedBlock, newBlock.NumberU64()-10) - node.BlockChannel <- newBlock + node.Consensus.BlockChannel(newBlock) break } else { retryCount++ diff --git a/node/service_setup.go b/node/service_setup.go index 89acaf8fb8..16c554ea8a 100644 --- a/node/service_setup.go +++ b/node/service_setup.go @@ -14,7 +14,7 @@ func (node *Node) RegisterValidatorServices() { // Register consensus service. node.serviceManager.Register( service.Consensus, - consensus.New(node.BlockChannel, node.Consensus, node.startConsensus), + consensus.New(node.Consensus, node.startConsensus), ) // Register new block service. node.serviceManager.Register( From 356cc14ed9100166566685cdcb9aacb7589b4816 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 01:11:36 +0800 Subject: [PATCH 073/420] Removed channels. --- api/service/consensus/service.go | 12 +-- consensus/consensus.go | 5 +- consensus/consensus_v2.go | 134 +++++++++++++++---------------- node/node.go | 5 +- node/node_handler.go | 4 +- node/service_setup.go | 2 +- 6 files changed, 76 insertions(+), 86 deletions(-) diff --git a/api/service/consensus/service.go b/api/service/consensus/service.go index e31ae71bbe..d85a072f2e 100644 --- a/api/service/consensus/service.go +++ b/api/service/consensus/service.go @@ -10,22 +10,19 @@ import ( type Service struct { consensus *consensus.Consensus stopChan chan struct{} - stoppedChan chan struct{} - startChan chan struct{} messageChan chan *msg_pb.Message } // New returns consensus service. -func New(consensus *consensus.Consensus, startChan chan struct{}) *Service { - return &Service{consensus: consensus, startChan: startChan} +func New(consensus *consensus.Consensus) *Service { + return &Service{consensus: consensus} } // Start starts consensus service. func (s *Service) Start() error { utils.Logger().Info().Msg("[consensus/service] Starting consensus service.") s.stopChan = make(chan struct{}) - s.stoppedChan = make(chan struct{}) - s.consensus.Start(s.stopChan, s.stoppedChan, s.startChan) + s.consensus.Start(s.stopChan) s.consensus.WaitForNewRandomness() return nil } @@ -33,8 +30,7 @@ func (s *Service) Start() error { // Stop stops consensus service. func (s *Service) Stop() error { utils.Logger().Info().Msg("Stopping consensus service.") - s.stopChan <- struct{}{} - <-s.stoppedChan + close(s.stopChan) utils.Logger().Info().Msg("Consensus service stopped.") return s.consensus.Close() } diff --git a/consensus/consensus.go b/consensus/consensus.go index dbfb39720e..5a131fac5d 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -133,8 +133,9 @@ type Consensus struct { dHelper *downloadHelper - // Flag only for initialization state. - start bool + // Both flags only for initialization state. + start bool + isInitialLeader bool } // Blockchain returns the blockchain. diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index d849898d60..2a133755b7 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -292,81 +292,28 @@ func (consensus *Consensus) BlockCommitSigs(blockNum uint64) ([]byte, error) { // Start waits for the next new block and run consensus func (consensus *Consensus) Start( - stopChan, stoppedChan, startChannel chan struct{}, + stopChan chan struct{}, ) { go func() { - toStart := make(chan struct{}, 1) - isInitialLeader := consensus.IsLeader() - if isInitialLeader { - consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Waiting for consensus start") - // send a signal to indicate it's ready to run consensus - // this signal is consumed by node object to create a new block and in turn trigger a new consensus on it - go func() { - <-startChannel - toStart <- struct{}{} - consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Send ReadySignal") - consensus.ReadySignal <- SyncProposal - }() - } consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Consensus started") - defer close(stoppedChan) - ticker := time.NewTicker(250 * time.Millisecond) - defer ticker.Stop() + go func() { + ticker := time.NewTicker(250 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-stopChan: + return + case <-ticker.C: + consensus.tick() + } + } + }() + consensus.consensusTimeout[timeoutBootstrap].Start() consensus.getLogger().Info().Msg("[ConsensusMainLoop] Start bootstrap timeout (only once)") // Set up next block due time. consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) - consensus.start = false - for { - select { - case <-toStart: - consensus.start = true - case <-ticker.C: - if !consensus.start && isInitialLeader { - continue - } - for k, v := range consensus.consensusTimeout { - // stop timer in listening mode - if consensus.current.Mode() == Listening { - v.Stop() - continue - } - - if consensus.current.Mode() == Syncing { - // never stop bootstrap timer here in syncing mode as it only starts once - // if it is stopped, bootstrap will be stopped and nodes - // can't start view change or join consensus - // the bootstrap timer will be stopped once consensus is reached or view change - // is succeeded - if k != timeoutBootstrap { - consensus.getLogger().Debug(). - Str("k", k.String()). - Str("Mode", consensus.current.Mode().String()). - Msg("[ConsensusMainLoop] consensusTimeout stopped!!!") - v.Stop() - continue - } - } - if !v.CheckExpire() { - continue - } - if k != timeoutViewChange { - consensus.getLogger().Warn().Msg("[ConsensusMainLoop] Ops Consensus Timeout!!!") - consensus.startViewChange() - break - } else { - consensus.getLogger().Warn().Msg("[ConsensusMainLoop] Ops View Change Timeout!!!") - consensus.startViewChange() - break - } - } - - case <-stopChan: - consensus.getLogger().Info().Msg("[ConsensusMainLoop] stopChan") - return - } - } }() if consensus.dHelper != nil { @@ -374,6 +321,15 @@ func (consensus *Consensus) Start( } } +func (consensus *Consensus) StartChannel() { + consensus.isInitialLeader = consensus.IsLeader() + if consensus.isInitialLeader { + consensus.start = true + consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Send ReadySignal") + consensus.ReadySignal <- SyncProposal + } +} + func (consensus *Consensus) syncReadyChan() { consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncReadyChan") if consensus.BlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { @@ -394,7 +350,6 @@ func (consensus *Consensus) syncReadyChan() { consensus.consensusTimeout[timeoutConsensus].Start() consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() } - } func (consensus *Consensus) syncNotReadyChan() { @@ -405,7 +360,48 @@ func (consensus *Consensus) syncNotReadyChan() { consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() } -// Close close the consensus. If current is in normal commit phase, wait until the commit +func (consensus *Consensus) tick() { + if !consensus.start && consensus.isInitialLeader { + return + } + for k, v := range consensus.consensusTimeout { + // stop timer in listening mode + if consensus.current.Mode() == Listening { + v.Stop() + continue + } + + if consensus.current.Mode() == Syncing { + // never stop bootstrap timer here in syncing mode as it only starts once + // if it is stopped, bootstrap will be stopped and nodes + // can't start view change or join consensus + // the bootstrap timer will be stopped once consensus is reached or view change + // is succeeded + if k != timeoutBootstrap { + consensus.getLogger().Debug(). + Str("k", k.String()). + Str("Mode", consensus.current.Mode().String()). + Msg("[ConsensusMainLoop] consensusTimeout stopped!!!") + v.Stop() + continue + } + } + if !v.CheckExpire() { + continue + } + if k != timeoutViewChange { + consensus.getLogger().Warn().Msg("[ConsensusMainLoop] Ops Consensus Timeout!!!") + consensus.startViewChange() + break + } else { + consensus.getLogger().Warn().Msg("[ConsensusMainLoop] Ops View Change Timeout!!!") + consensus.startViewChange() + break + } + } +} + +// Close closes the consensus. If current is in normal commit phase, wait until the commit // phase end. func (consensus *Consensus) Close() error { if consensus.dHelper != nil { diff --git a/node/node.go b/node/node.go index a5c1288876..ba398d6289 100644 --- a/node/node.go +++ b/node/node.go @@ -126,9 +126,7 @@ type Node struct { serviceManager *service.Manager ContractDeployerCurrentNonce uint64 // The nonce of the deployer contract at current block ContractAddresses []common.Address - // Channel to notify consensus service to really start consensus - startConsensus chan struct{} - HarmonyConfig *harmonyconfig.HarmonyConfig + HarmonyConfig *harmonyconfig.HarmonyConfig // node configuration, including group ID, shard ID, etc NodeConfig *nodeconfig.ConfigType // Chain configuration. @@ -1130,7 +1128,6 @@ func New( Msg("Genesis block hash") // Setup initial state of syncing. node.peerRegistrationRecord = map[string]*syncConfig{} - node.startConsensus = make(chan struct{}) // Broadcast double-signers reported by consensus if node.Consensus != nil { go func() { diff --git a/node/node_handler.go b/node/node_handler.go index a34d73e94d..d15b41eb5a 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -434,7 +434,7 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block) error { return nil } -// BootstrapConsensus is the a goroutine to check number of peers and start the consensus +// BootstrapConsensus is a goroutine to check number of peers and start the consensus func (node *Node) BootstrapConsensus() error { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() @@ -464,7 +464,7 @@ func (node *Node) BootstrapConsensus() error { return ctx.Err() case <-enoughMinPeers: go func() { - node.startConsensus <- struct{}{} + node.Consensus.StartChannel() }() return nil } diff --git a/node/service_setup.go b/node/service_setup.go index 16c554ea8a..a2518110c1 100644 --- a/node/service_setup.go +++ b/node/service_setup.go @@ -14,7 +14,7 @@ func (node *Node) RegisterValidatorServices() { // Register consensus service. node.serviceManager.Register( service.Consensus, - consensus.New(node.Consensus, node.startConsensus), + consensus.New(node.Consensus), ) // Register new block service. node.serviceManager.Register( From e264e11bdf043f3370790c348bb913ba6ffa2889 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 02:19:35 +0800 Subject: [PATCH 074/420] Removed view change locks. --- consensus/view_change.go | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/consensus/view_change.go b/consensus/view_change.go index 3bd0c181f2..0bfdd92215 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -2,7 +2,6 @@ package consensus import ( "math/big" - "sync" "time" "github.com/harmony-one/harmony/internal/chain" @@ -25,26 +24,21 @@ const MaxViewIDDiff = 249 // State contains current mode and current viewID type State struct { - mode Mode - modeMux sync.RWMutex + mode Mode // current view id in normal mode // it changes per successful consensus blockViewID uint64 - cViewMux sync.RWMutex // view changing id is used during view change mode // it is the next view id viewChangingID uint64 - viewMux sync.RWMutex isBackup bool } // Mode return the current node mode func (pm *State) Mode() Mode { - pm.modeMux.RLock() - defer pm.modeMux.RUnlock() return pm.mode } @@ -54,22 +48,16 @@ func (pm *State) SetMode(s Mode) { s = NormalBackup } - pm.modeMux.Lock() - defer pm.modeMux.Unlock() pm.mode = s } // GetCurBlockViewID return the current view id func (pm *State) GetCurBlockViewID() uint64 { - pm.cViewMux.RLock() - defer pm.cViewMux.RUnlock() return pm.blockViewID } // SetCurBlockViewID sets the current view id func (pm *State) SetCurBlockViewID(viewID uint64) uint64 { - pm.cViewMux.Lock() - defer pm.cViewMux.Unlock() pm.blockViewID = viewID return pm.blockViewID } @@ -77,26 +65,18 @@ func (pm *State) SetCurBlockViewID(viewID uint64) uint64 { // GetViewChangingID return the current view changing id // It is meaningful during view change mode func (pm *State) GetViewChangingID() uint64 { - pm.viewMux.RLock() - defer pm.viewMux.RUnlock() return pm.viewChangingID } // SetViewChangingID set the current view changing id // It is meaningful during view change mode func (pm *State) SetViewChangingID(id uint64) { - pm.viewMux.Lock() - defer pm.viewMux.Unlock() pm.viewChangingID = id } // GetViewChangeDuraion return the duration of the current view change // It increase in the power of difference betweeen view changing ID and current view ID func (pm *State) GetViewChangeDuraion() time.Duration { - pm.viewMux.RLock() - pm.cViewMux.RLock() - defer pm.viewMux.RUnlock() - defer pm.cViewMux.RUnlock() diff := int64(pm.viewChangingID - pm.blockViewID) return time.Duration(diff * diff * int64(viewChangeDuration)) } From 5175ec62256999de053e0bf2ecfe13ad3c7468c2 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 02:25:09 +0800 Subject: [PATCH 075/420] Removed timers locks. --- internal/utils/timer.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/internal/utils/timer.go b/internal/utils/timer.go index 3502d68ecc..2e8a77667b 100644 --- a/internal/utils/timer.go +++ b/internal/utils/timer.go @@ -1,7 +1,6 @@ package utils import ( - "sync" "time" ) @@ -20,7 +19,6 @@ type Timeout struct { state TimeoutState d time.Duration start time.Time - mu sync.Mutex } // NewTimeout creates a new timeout class @@ -31,24 +29,18 @@ func NewTimeout(d time.Duration) *Timeout { // Start starts the timeout clock func (timeout *Timeout) Start() { - timeout.mu.Lock() timeout.state = Active timeout.start = time.Now() - timeout.mu.Unlock() } // Stop stops the timeout clock func (timeout *Timeout) Stop() { - timeout.mu.Lock() timeout.state = Inactive timeout.start = time.Now() - timeout.mu.Unlock() } // CheckExpire checks whether the timeout is reached/expired func (timeout *Timeout) CheckExpire() bool { - timeout.mu.Lock() - defer timeout.mu.Unlock() if timeout.state == Active && time.Since(timeout.start) > timeout.d { timeout.state = Expired } @@ -60,23 +52,17 @@ func (timeout *Timeout) CheckExpire() bool { // Duration returns the duration period of timeout func (timeout *Timeout) Duration() time.Duration { - timeout.mu.Lock() - defer timeout.mu.Unlock() return timeout.d } // SetDuration set new duration for the timer func (timeout *Timeout) SetDuration(nd time.Duration) { - timeout.mu.Lock() timeout.d = nd - timeout.mu.Unlock() } // IsActive checks whether timeout clock is active; // A timeout is active means it's not stopped caused by stop // and also not expired with time elapses longer than duration from start func (timeout *Timeout) IsActive() bool { - timeout.mu.Lock() - defer timeout.mu.Unlock() return timeout.state == Active } From ce18c59d4b9979a28062e87370c898d057a4484b Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 02:28:03 +0800 Subject: [PATCH 076/420] Removed fbft locks. --- consensus/consensus_fbft.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/consensus/consensus_fbft.go b/consensus/consensus_fbft.go index 313abf0615..29c9714d35 100644 --- a/consensus/consensus_fbft.go +++ b/consensus/consensus_fbft.go @@ -1,9 +1,6 @@ package consensus -import "sync" - type LockedFBFTPhase struct { - mu sync.Mutex phase FBFTPhase } @@ -14,14 +11,10 @@ func NewLockedFBFTPhase(initialPhrase FBFTPhase) *LockedFBFTPhase { } func (a *LockedFBFTPhase) Set(phrase FBFTPhase) { - a.mu.Lock() a.phase = phrase - a.mu.Unlock() } func (a *LockedFBFTPhase) Get() FBFTPhase { - a.mu.Lock() - defer a.mu.Unlock() return a.phase } From e4ba190e633f73d6795ad74632643e16b353ee23 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 02:38:12 +0800 Subject: [PATCH 077/420] Removed multiSigMutex locks. --- consensus/consensus.go | 1 - consensus/consensus_service.go | 3 +-- consensus/fbft_log.go | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 5a131fac5d..a69bf941bb 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -61,7 +61,6 @@ type Consensus struct { commitBitmap *bls_cosi.Mask multiSigBitmap *bls_cosi.Mask // Bitmap for parsing multisig bitmap from validators - multiSigMutex sync.RWMutex // Registry for services. registry *registry.Registry diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 1844f22687..5534b9bc93 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -138,9 +138,8 @@ func (consensus *Consensus) UpdateBitmaps() { multiSigBitmap, _ := bls_cosi.NewMask(members, nil) consensus.prepareBitmap = prepareBitmap consensus.commitBitmap = commitBitmap - consensus.multiSigMutex.Lock() consensus.multiSigBitmap = multiSigBitmap - consensus.multiSigMutex.Unlock() + } // ResetState resets the state of the consensus diff --git a/consensus/fbft_log.go b/consensus/fbft_log.go index 1a59665729..dead27f1a7 100644 --- a/consensus/fbft_log.go +++ b/consensus/fbft_log.go @@ -360,9 +360,7 @@ func (consensus *Consensus) ParseFBFTMessage(msg *msg_pb.Message) (*FBFTMessage, copy(pbftMsg.SenderPubkeys[0].Bytes[:], consensusMsg.SenderPubkey[:]) } else { // else, it should be a multi-key message where the bitmap is populated - consensus.multiSigMutex.RLock() pubKeys, err := consensus.multiSigBitmap.GetSignedPubKeysFromBitmap(pbftMsg.SenderPubkeyBitmap) - consensus.multiSigMutex.RUnlock() if err != nil { return nil, err } From da0b0cef7e60ed6b7a41213590e8bfc3545db58f Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 02:52:44 +0800 Subject: [PATCH 078/420] Removed leader locks. --- consensus/consensus.go | 5 +--- consensus/consensus_service.go | 7 ------ consensus/consensus_v2.go | 43 +++------------------------------- consensus/view_change.go | 7 +----- 4 files changed, 5 insertions(+), 57 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index a69bf941bb..872eac2a74 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -66,8 +66,7 @@ type Consensus struct { registry *registry.Registry // Minimal number of peers in the shard // If the number of validators is less than minPeers, the consensus won't start - MinPeers int - pubKeyLock sync.Mutex + MinPeers int // private/public keys of current node priKey multibls.PrivateKeys // the publickey of leader @@ -177,8 +176,6 @@ func (consensus *Consensus) GetPublicKeys() multibls.PublicKeys { } func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() return consensus.LeaderPubKey } diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 5534b9bc93..3b46f54b7e 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -73,8 +73,6 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { - // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. - consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) consensus.getLogger().Info().Msg("My Committee updated") for i := range pubKeys { @@ -93,7 +91,6 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi consensus.getLogger().Error(). Msg("[UpdatePublicKeys] Participants is empty") } - consensus.pubKeyLock.Unlock() // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. consensus.UpdateBitmaps() @@ -393,9 +390,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { consensus.getLogger().Info(). Str("leaderPubKey", leaderPubKey.Bytes.Hex()). Msg("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain") - consensus.pubKeyLock.Lock() consensus.LeaderPubKey = leaderPubKey - consensus.pubKeyLock.Unlock() } } @@ -434,9 +429,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { - consensus.pubKeyLock.Lock() obj := consensus.LeaderPubKey.Object - consensus.pubKeyLock.Unlock() for _, key := range consensus.priKey { if key.Pub.Object.IsEqual(obj) { return true diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 2a133755b7..ad3436eaaa 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -744,52 +744,15 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { // SetupForNewConsensus sets the state for new consensus func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) - curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) // first view id is going to be 2. - prev := consensus.GetLeaderPubKey() - if consensus.Blockchain.Config().IsLeaderRotation(blk.Epoch()) { - epochBlockViewID, err := consensus.getEpochFirstBlockViewID(blk.Epoch()) - if err != nil { - consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", blk.Epoch().Uint64()) - return - } - if epochBlockViewID > curBlockViewID { - consensus.getLogger().Error().Msg("[SetupForNewConsensus] Epoch block viewID is greater than current block viewID") - return - } - - diff := curBlockViewID - epochBlockViewID - - pps := consensus.Decider.Participants() - idx := (int(diff) / 3) % len(pps) - consensus.pubKeyLock.Lock() - fmt.Println("(int(diff)/3)%len(pps) == ", idx) - consensus.LeaderPubKey = &pps[idx] - fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) - consensus.pubKeyLock.Unlock() - if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { - // leader changed - go func() { - fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - consensus.ReadySignal <- SyncProposal - - }() - } - } else { - fmt.Printf("SetupForNewConsensus0 :%d future v%d new: %s prev: %s %v\n", utils.GetPort(), curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] - consensus.pubKeyLock.Unlock() - } - - } + consensus.SetCurBlockViewID(committedMsg.ViewID + 1) + consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] var epoch *big.Int if blk.IsLastBlockInEpoch() { epoch = new(big.Int).Add(blk.Epoch(), common.Big1) } else { epoch = blk.Epoch() } - if consensus.Blockchain().Config().IsLeaderRotation(epoch) { + if consensus.Blockchain.Config().IsLeaderRotation(epoch) { consensus.rotateLeader(epoch) } diff --git a/consensus/view_change.go b/consensus/view_change.go index 0bfdd92215..1267684d66 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -251,10 +251,7 @@ func (consensus *Consensus) startViewChange() { // aganist the consensus.LeaderPubKey variable. // Ideally, we shall use another variable to keep track of the // leader pubkey in viewchange mode - consensus.pubKeyLock.Lock() - lpk := consensus.getNextLeaderKey(nextViewID) - consensus.LeaderPubKey = lpk - consensus.pubKeyLock.Unlock() + consensus.LeaderPubKey = consensus.getNextLeaderKey(nextViewID) consensus.getLogger().Warn(). Uint64("nextViewID", nextViewID). @@ -549,9 +546,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // newView message verified success, override my state consensus.SetViewIDs(recvMsg.ViewID) - consensus.pubKeyLock.Lock() consensus.LeaderPubKey = senderKey - consensus.pubKeyLock.Unlock() consensus.ResetViewChangeState() consensus.msgSender.StopRetry(msg_pb.MessageType_VIEWCHANGE) From 08297fff154ee0d35cefa20fc8e57574dad45c9b Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 16:51:24 +0800 Subject: [PATCH 079/420] Removed additional locks and isViewChange. --- consensus/checks.go | 4 +-- consensus/consensus_service.go | 2 +- consensus/consensus_v2.go | 10 +++++-- consensus/fbft_log.go | 51 +--------------------------------- consensus/validator.go | 4 +-- consensus/view_change.go | 8 ++---- 6 files changed, 17 insertions(+), 62 deletions(-) diff --git a/consensus/checks.go b/consensus/checks.go index c44a58da39..9e095a45c4 100644 --- a/consensus/checks.go +++ b/consensus/checks.go @@ -86,7 +86,7 @@ func (consensus *Consensus) onAnnounceSanityChecks(recvMsg *FBFTMessage) bool { Str("recvMsg", recvMsg.String()). Str("LeaderKey", consensus.LeaderPubKey.Bytes.Hex()). Msg("[OnAnnounce] Leader is malicious") - if consensus.IsViewChangingMode() { + if consensus.isViewChangingMode() { consensus.getLogger().Debug().Msg( "[OnAnnounce] Already in ViewChanging mode, conflicing announce, doing noop", ) @@ -161,7 +161,7 @@ func (consensus *Consensus) onViewChangeSanityCheck(recvMsg *FBFTMessage) bool { Msg("[onViewChangeSanityCheck] MsgBlockNum is different from my BlockNumber") return false } - if consensus.IsViewChangingMode() && + if consensus.isViewChangingMode() && consensus.GetCurBlockViewID() > recvMsg.ViewID { consensus.getLogger().Debug().Uint64("curBlockViewID", consensus.GetCurBlockViewID()). Uint64("msgViewID", recvMsg.ViewID). diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 3b46f54b7e..ee0e6026c6 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -97,7 +97,7 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi consensus.ResetState() // do not reset view change state if it is in view changing mode - if !consensus.IsViewChangingMode() { + if !consensus.isViewChangingMode() { consensus.ResetViewChangeState() } return consensus.Decider.ParticipantsCount() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index ad3436eaaa..710c4e72b8 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -47,6 +47,12 @@ const ( // IsViewChangingMode return true if curernt mode is viewchanging func (consensus *Consensus) IsViewChangingMode() bool { + //consensus.mutex.RLock() + //defer consensus.mutex.RUnlock() + return consensus.isViewChangingMode() +} + +func (consensus *Consensus) isViewChangingMode() bool { return consensus.current.Mode() == ViewChanging } @@ -56,7 +62,7 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb // in order to avoid possible trap forever but drop PREPARE and COMMIT // which are message types specifically for a node acting as leader // so we just ignore those messages - if consensus.IsViewChangingMode() && + if consensus.isViewChangingMode() && (msg.Type == msg_pb.MessageType_PREPARE || msg.Type == msg_pb.MessageType_COMMIT) { return nil @@ -784,7 +790,7 @@ func (consensus *Consensus) postCatchup(initBN uint64) { consensus.switchPhase("TryCatchup", FBFTAnnounce) } // catch up and skip from view change trap - if initBN < consensus.BlockNum() && consensus.IsViewChangingMode() { + if initBN < consensus.BlockNum() && consensus.isViewChangingMode() { consensus.current.SetMode(Normal) consensus.consensusTimeout[timeoutViewChange].Stop() } diff --git a/consensus/fbft_log.go b/consensus/fbft_log.go index dead27f1a7..4743085ab6 100644 --- a/consensus/fbft_log.go +++ b/consensus/fbft_log.go @@ -3,7 +3,6 @@ package consensus import ( "encoding/binary" "fmt" - "sync" "github.com/ethereum/go-ethereum/common" bls_core "github.com/harmony-one/bls/ffi/go/bls" @@ -102,10 +101,7 @@ func (m *FBFTMessage) id() fbftMsgID { type FBFTLog struct { blocks map[common.Hash]*types.Block // store blocks received in FBFT verifiedBlocks map[common.Hash]struct{} // store block hashes for blocks that has already been verified - blockLock sync.RWMutex - - messages map[fbftMsgID]*FBFTMessage // store messages received in FBFT - msgLock sync.RWMutex + messages map[fbftMsgID]*FBFTMessage // store messages received in FBFT } // NewFBFTLog returns new instance of FBFTLog @@ -120,42 +116,27 @@ func NewFBFTLog() *FBFTLog { // AddBlock add a new block into the log func (log *FBFTLog) AddBlock(block *types.Block) { - log.blockLock.Lock() - defer log.blockLock.Unlock() - log.blocks[block.Hash()] = block } // MarkBlockVerified marks the block as verified func (log *FBFTLog) MarkBlockVerified(block *types.Block) { - log.blockLock.Lock() - defer log.blockLock.Unlock() - log.verifiedBlocks[block.Hash()] = struct{}{} } // IsBlockVerified checks whether the block is verified func (log *FBFTLog) IsBlockVerified(hash common.Hash) bool { - log.blockLock.RLock() - defer log.blockLock.RUnlock() - _, exist := log.verifiedBlocks[hash] return exist } // GetBlockByHash returns the block matches the given block hash func (log *FBFTLog) GetBlockByHash(hash common.Hash) *types.Block { - log.blockLock.RLock() - defer log.blockLock.RUnlock() - return log.blocks[hash] } // GetBlocksByNumber returns the blocks match the given block number func (log *FBFTLog) GetBlocksByNumber(number uint64) []*types.Block { - log.blockLock.RLock() - defer log.blockLock.RUnlock() - var blocks []*types.Block for _, block := range log.blocks { if block.NumberU64() == number { @@ -167,9 +148,6 @@ func (log *FBFTLog) GetBlocksByNumber(number uint64) []*types.Block { // DeleteBlocksLessThan deletes blocks less than given block number func (log *FBFTLog) DeleteBlocksLessThan(number uint64) { - log.blockLock.Lock() - defer log.blockLock.Unlock() - for h, block := range log.blocks { if block.NumberU64() < number { delete(log.blocks, h) @@ -180,9 +158,6 @@ func (log *FBFTLog) DeleteBlocksLessThan(number uint64) { // DeleteBlockByNumber deletes block of specific number func (log *FBFTLog) DeleteBlockByNumber(number uint64) { - log.blockLock.Lock() - defer log.blockLock.Unlock() - for h, block := range log.blocks { if block.NumberU64() == number { delete(log.blocks, h) @@ -193,9 +168,6 @@ func (log *FBFTLog) DeleteBlockByNumber(number uint64) { // DeleteMessagesLessThan deletes messages less than given block number func (log *FBFTLog) DeleteMessagesLessThan(number uint64) { - log.msgLock.Lock() - defer log.msgLock.Unlock() - for h, msg := range log.messages { if msg.BlockNum < number { delete(log.messages, h) @@ -205,9 +177,6 @@ func (log *FBFTLog) DeleteMessagesLessThan(number uint64) { // AddVerifiedMessage adds a signature verified pbft message into the log func (log *FBFTLog) AddVerifiedMessage(msg *FBFTMessage) { - log.msgLock.Lock() - defer log.msgLock.Unlock() - msg.Verified = true log.messages[msg.id()] = msg @@ -215,9 +184,6 @@ func (log *FBFTLog) AddVerifiedMessage(msg *FBFTMessage) { // AddNotVerifiedMessage adds a not signature verified pbft message into the log func (log *FBFTLog) AddNotVerifiedMessage(msg *FBFTMessage) { - log.msgLock.Lock() - defer log.msgLock.Unlock() - msg.Verified = false log.messages[msg.id()] = msg @@ -225,9 +191,6 @@ func (log *FBFTLog) AddNotVerifiedMessage(msg *FBFTMessage) { // GetNotVerifiedCommittedMessages returns not verified committed pbft messages with matching blockNum, viewID and blockHash func (log *FBFTLog) GetNotVerifiedCommittedMessages(blockNum uint64, viewID uint64, blockHash common.Hash) []*FBFTMessage { - log.msgLock.RLock() - defer log.msgLock.RUnlock() - var found []*FBFTMessage for _, msg := range log.messages { if msg.MessageType == msg_pb.MessageType_COMMITTED && msg.BlockNum == blockNum && msg.ViewID == viewID && msg.BlockHash == blockHash && !msg.Verified { @@ -239,9 +202,6 @@ func (log *FBFTLog) GetNotVerifiedCommittedMessages(blockNum uint64, viewID uint // GetMessagesByTypeSeqViewHash returns pbft messages with matching type, blockNum, viewID and blockHash func (log *FBFTLog) GetMessagesByTypeSeqViewHash(typ msg_pb.MessageType, blockNum uint64, viewID uint64, blockHash common.Hash) []*FBFTMessage { - log.msgLock.RLock() - defer log.msgLock.RUnlock() - var found []*FBFTMessage for _, msg := range log.messages { if msg.MessageType == typ && msg.BlockNum == blockNum && msg.ViewID == viewID && msg.BlockHash == blockHash && msg.Verified { @@ -253,9 +213,6 @@ func (log *FBFTLog) GetMessagesByTypeSeqViewHash(typ msg_pb.MessageType, blockNu // GetMessagesByTypeSeq returns pbft messages with matching type, blockNum func (log *FBFTLog) GetMessagesByTypeSeq(typ msg_pb.MessageType, blockNum uint64) []*FBFTMessage { - log.msgLock.RLock() - defer log.msgLock.RUnlock() - var found []*FBFTMessage for _, msg := range log.messages { if msg.MessageType == typ && msg.BlockNum == blockNum && msg.Verified { @@ -267,9 +224,6 @@ func (log *FBFTLog) GetMessagesByTypeSeq(typ msg_pb.MessageType, blockNum uint64 // GetMessagesByTypeSeqHash returns pbft messages with matching type, blockNum func (log *FBFTLog) GetMessagesByTypeSeqHash(typ msg_pb.MessageType, blockNum uint64, blockHash common.Hash) []*FBFTMessage { - log.msgLock.RLock() - defer log.msgLock.RUnlock() - var found []*FBFTMessage for _, msg := range log.messages { if msg.MessageType == typ && msg.BlockNum == blockNum && msg.BlockHash == blockHash && msg.Verified { @@ -305,9 +259,6 @@ func (log *FBFTLog) HasMatchingViewPrepared(blockNum uint64, viewID uint64, bloc // GetMessagesByTypeSeqView returns pbft messages with matching type, blockNum and viewID func (log *FBFTLog) GetMessagesByTypeSeqView(typ msg_pb.MessageType, blockNum uint64, viewID uint64) []*FBFTMessage { - log.msgLock.RLock() - defer log.msgLock.RUnlock() - var found []*FBFTMessage for _, msg := range log.messages { if msg.MessageType != typ || msg.BlockNum != blockNum || msg.ViewID != viewID && msg.Verified { diff --git a/consensus/validator.go b/consensus/validator.go index 013769395c..3e958fef26 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -47,7 +47,7 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { consensus.blockHash = recvMsg.BlockHash // we have already added message and block, skip check viewID // and send prepare message if is in ViewChanging mode - if consensus.IsViewChangingMode() { + if consensus.isViewChangingMode() { consensus.getLogger().Debug(). Msg("[OnAnnounce] Still in ViewChanging Mode, Exiting !!") return @@ -392,7 +392,7 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { return } - if consensus.IsViewChangingMode() { + if consensus.isViewChangingMode() { consensus.getLogger().Info().Msg("[OnCommitted] Still in ViewChanging mode, Exiting!!") return } diff --git a/consensus/view_change.go b/consensus/view_change.go index 1267684d66..aff25c2d15 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -237,8 +237,6 @@ func (consensus *Consensus) startViewChange() { if consensus.disableViewChange || consensus.IsBackup() { return } - consensus.mutex.Lock() - defer consensus.mutex.Unlock() consensus.consensusTimeout[timeoutConsensus].Stop() consensus.consensusTimeout[timeoutBootstrap].Stop() @@ -300,7 +298,7 @@ func (consensus *Consensus) startViewChange() { // startNewView stops the current view change func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.PrivateKeyWrapper, reset bool) error { - if !consensus.IsViewChangingMode() { + if !consensus.isViewChangingMode() { return errors.New("not in view changing mode anymore") } @@ -421,7 +419,7 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { } // received enough view change messages, change state to normal consensus - if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) && consensus.IsViewChangingMode() { + if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) && consensus.isViewChangingMode() { // no previous prepared message, go straight to normal mode // and start proposing new block if consensus.vc.IsM1PayloadEmpty() { @@ -537,7 +535,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { } } - if !consensus.IsViewChangingMode() { + if !consensus.isViewChangingMode() { consensus.getLogger().Info().Msg("Not in ViewChanging Mode.") return } From b605d4ad17db19f6ae770f08f88dcde9ca8cdfd2 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 18:37:20 +0800 Subject: [PATCH 080/420] Added locks detected by race. --- consensus/consensus_v2.go | 5 ++++- scripts/go_executable_build.sh | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 710c4e72b8..4575e197ef 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -310,16 +310,19 @@ func (consensus *Consensus) Start( case <-stopChan: return case <-ticker.C: + consensus.mutex.Lock() consensus.tick() + consensus.mutex.Unlock() } } }() + consensus.mutex.Lock() consensus.consensusTimeout[timeoutBootstrap].Start() consensus.getLogger().Info().Msg("[ConsensusMainLoop] Start bootstrap timeout (only once)") - // Set up next block due time. consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) + consensus.mutex.Unlock() }() if consensus.dHelper != nil { diff --git a/scripts/go_executable_build.sh b/scripts/go_executable_build.sh index 2c1cc99f0e..1aae7c9df7 100755 --- a/scripts/go_executable_build.sh +++ b/scripts/go_executable_build.sh @@ -120,6 +120,7 @@ function build_only if [ "$STATIC" == "true" ]; then env GOOS=$GOOS GOARCH=$GOARCH go build $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY} -w -extldflags \"-static -lm\"" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} else +# env GOOS=$GOOS GOARCH=$GOARCH go build -race $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY}" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} env GOOS=$GOOS GOARCH=$GOARCH go build $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY}" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} fi fi From 2cae02d4c04b1634a35d82dc83ff80a3e2466a50 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 18:37:42 +0800 Subject: [PATCH 081/420] Added locks detected by race. --- scripts/go_executable_build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/go_executable_build.sh b/scripts/go_executable_build.sh index 1aae7c9df7..2c1cc99f0e 100755 --- a/scripts/go_executable_build.sh +++ b/scripts/go_executable_build.sh @@ -120,7 +120,6 @@ function build_only if [ "$STATIC" == "true" ]; then env GOOS=$GOOS GOARCH=$GOARCH go build $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY} -w -extldflags \"-static -lm\"" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} else -# env GOOS=$GOOS GOARCH=$GOARCH go build -race $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY}" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} env GOOS=$GOOS GOARCH=$GOARCH go build $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY}" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} fi fi From d533d375e5f74ced228842d8260015d3b0bddb04 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 19:38:31 +0800 Subject: [PATCH 082/420] Locks for start. --- consensus/consensus_v2.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 4575e197ef..4db16a3725 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -331,12 +331,16 @@ func (consensus *Consensus) Start( } func (consensus *Consensus) StartChannel() { + consensus.mutex.Lock() consensus.isInitialLeader = consensus.IsLeader() if consensus.isInitialLeader { consensus.start = true consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Send ReadySignal") + consensus.mutex.Unlock() consensus.ReadySignal <- SyncProposal + return } + consensus.mutex.Unlock() } func (consensus *Consensus) syncReadyChan() { From 6cfe6b32a1c02406a990fe93cc0e30bb1f21adc0 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 20:03:09 +0800 Subject: [PATCH 083/420] Removed additional logs. --- consensus/leader.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/consensus/leader.go b/consensus/leader.go index 422508762b..6b176d27ae 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -26,12 +26,8 @@ func (consensus *Consensus) announce(block *types.Block) { return } - //// Lock Write - Start - consensus.mutex.Lock() copy(consensus.blockHash[:], blockHash[:]) consensus.block = encodedBlock // Must set block bytes before consensus.construct() - consensus.mutex.Unlock() - //// Lock Write - End key, err := consensus.GetConsensusLeaderPrivateKey() if err != nil { From 883ac07baa1f590d7d272ba7c178c0ac7cc9be10 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 20:35:59 +0800 Subject: [PATCH 084/420] Removed additional locks. --- consensus/consensus_service.go | 1 - consensus/consensus_v2.go | 2 -- 2 files changed, 3 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index ee0e6026c6..9a171ea1d0 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -493,7 +493,6 @@ func (consensus *Consensus) switchPhase(subject string, desired FBFTPhase) { Str("switchPhase:", subject) consensus.phase.Set(desired) - return } var ( diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 4db16a3725..876613d819 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -47,8 +47,6 @@ const ( // IsViewChangingMode return true if curernt mode is viewchanging func (consensus *Consensus) IsViewChangingMode() bool { - //consensus.mutex.RLock() - //defer consensus.mutex.RUnlock() return consensus.isViewChangingMode() } From 97b4c735dc7467ca9b9b0d42367c08a686740b8a Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 3 Nov 2022 01:58:08 +0800 Subject: [PATCH 085/420] Removed additional locks. --- consensus/consensus.go | 6 ++++-- consensus/consensus_service.go | 2 +- consensus/validator.go | 10 ---------- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 872eac2a74..b24a303ccf 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -84,8 +84,6 @@ type Consensus struct { IgnoreViewIDCheck *abool.AtomicBool // consensus mutex mutex sync.RWMutex - // mutex for verify new block - verifyBlockMutex sync.Mutex // ViewChange struct vc *viewChange // Signal channel for proposing a new block and start new consensus @@ -172,6 +170,10 @@ func (consensus *Consensus) VdfSeedSize() int { // GetPublicKeys returns the public keys func (consensus *Consensus) GetPublicKeys() multibls.PublicKeys { + return consensus.getPublicKeys() +} + +func (consensus *Consensus) getPublicKeys() multibls.PublicKeys { return consensus.priKey.GetPublicKeys() } diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 9a171ea1d0..aec2e2b94b 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -396,7 +396,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { for _, key := range pubKeys { // in committee - myPubKeys := consensus.GetPublicKeys() + myPubKeys := consensus.getPublicKeys() if myPubKeys.Contains(key.Object) { if hasError { consensus.getLogger().Error(). diff --git a/consensus/validator.go b/consensus/validator.go index 3e958fef26..ffdcedb354 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -79,10 +79,6 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { } func (consensus *Consensus) validateNewBlock(recvMsg *FBFTMessage) (*types.Block, error) { - // Lock to prevent race condition between announce and prepare - consensus.verifyBlockMutex.Lock() - defer consensus.verifyBlockMutex.Unlock() - if consensus.FBFTLog.IsBlockVerified(recvMsg.BlockHash) { var blockObj *types.Block @@ -188,12 +184,6 @@ func (consensus *Consensus) sendCommitMessages(blockObj *types.Block) { // if onPrepared accepts the prepared message from the leader, then // it will send a COMMIT message for the leader to receive on the network. func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { - if consensus.ShardID == 0 { - //fmt.Println("onPrepared", recvMsg.BlockNum) - } - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - consensus.getLogger().Info(). Uint64("MsgBlockNum", recvMsg.BlockNum). Uint64("MsgViewID", recvMsg.ViewID). From 21095603b9248fab5cf98a7757b75508b1ff0a07 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sun, 6 Nov 2022 22:47:14 +0700 Subject: [PATCH 086/420] Make func private. --- consensus/consensus_service.go | 6 +++--- consensus/consensus_v2.go | 2 +- consensus/view_change.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index aec2e2b94b..bf420177cc 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -94,7 +94,7 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. consensus.UpdateBitmaps() - consensus.ResetState() + consensus.resetState() // do not reset view change state if it is in view changing mode if !consensus.isViewChangingMode() { @@ -140,7 +140,7 @@ func (consensus *Consensus) UpdateBitmaps() { } // ResetState resets the state of the consensus -func (consensus *Consensus) ResetState() { +func (consensus *Consensus) resetState() { consensus.switchPhase("ResetState", FBFTAnnounce) consensus.blockHash = [32]byte{} @@ -520,7 +520,7 @@ func (consensus *Consensus) selfCommit(payload []byte) error { } // Have to keep the block hash so the leader can finish the commit phase of prepared block - consensus.ResetState() + consensus.resetState() copy(consensus.blockHash[:], blockHash[:]) consensus.switchPhase("selfCommit", FBFTCommit) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 876613d819..763214c035 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -772,7 +772,7 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.SetMode(consensus.UpdateConsensusInformation()) } consensus.FBFTLog.PruneCacheBeforeBlock(blk.NumberU64()) - consensus.ResetState() + consensus.resetState() } func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, error) { diff --git a/consensus/view_change.go b/consensus/view_change.go index aff25c2d15..95ccbb97aa 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -338,7 +338,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri // TODO: consider make ResetState unified and only called in one place like finalizeCommit() if reset { - consensus.ResetState() + consensus.resetState() } consensus.SetLeaderPubKey(newLeaderPriKey.Pub) @@ -554,7 +554,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.sendCommitMessages(preparedBlock) consensus.switchPhase("onNewView", FBFTCommit) } else { - consensus.ResetState() + consensus.resetState() consensus.getLogger().Info().Msg("onNewView === announce") } consensus.getLogger().Info(). From f7ec5c47f946b913bb50bda5cd76e7514d9dbeac Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sun, 6 Nov 2022 23:42:26 +0700 Subject: [PATCH 087/420] Make VerifyBlock private. --- consensus/consensus.go | 4 ++-- consensus/consensus_v2.go | 2 +- consensus/validator.go | 9 ++++++--- consensus/view_change_msg.go | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index b24a303ccf..887b4744e9 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -140,7 +140,7 @@ func (consensus *Consensus) Blockchain() core.BlockChain { } // VerifyBlock is a function used to verify the block and keep trace of verified blocks. -func (consensus *Consensus) VerifyBlock(block *types.Block) error { +func (consensus *Consensus) verifyBlock(block *types.Block) error { if !consensus.FBFTLog.IsBlockVerified(block.Hash()) { if err := consensus.BlockVerifier(block); err != nil { return errors.Errorf("Block verification failed: %s", err) @@ -217,7 +217,7 @@ func (consensus *Consensus) GetConsensusLeaderPrivateKey() (*bls.PrivateKeyWrapp // SetBlockVerifier sets the block verifier func (consensus *Consensus) SetBlockVerifier(verifier VerifyBlockFunc) { consensus.BlockVerifier = verifier - consensus.vc.SetVerifyBlock(consensus.VerifyBlock) + consensus.vc.SetVerifyBlock(consensus.verifyBlock) } func (consensus *Consensus) IsBackup() bool { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 763214c035..18feeae286 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -662,7 +662,7 @@ func (consensus *Consensus) tryCatchup() error { } blk.SetCurrentCommitSig(msg.Payload) - if err := consensus.VerifyBlock(blk); err != nil { + if err := consensus.verifyBlock(blk); err != nil { consensus.getLogger().Err(err).Msg("[TryCatchup] failed block verifier") return err } diff --git a/consensus/validator.go b/consensus/validator.go index ffdcedb354..b33ac6d688 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -42,8 +42,6 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Uint64("MsgBlockNum", recvMsg.BlockNum). Msg("[OnAnnounce] Announce message Added") consensus.FBFTLog.AddVerifiedMessage(recvMsg) - consensus.mutex.Lock() - defer consensus.mutex.Unlock() consensus.blockHash = recvMsg.BlockHash // we have already added message and block, skip check viewID // and send prepare message if is in ViewChanging mode @@ -78,6 +76,11 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { } } +func (consensus *Consensus) ValidateNewBlock(recvMsg *FBFTMessage) (*types.Block, error) { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + return consensus.validateNewBlock(recvMsg) +} func (consensus *Consensus) validateNewBlock(recvMsg *FBFTMessage) (*types.Block, error) { if consensus.FBFTLog.IsBlockVerified(recvMsg.BlockHash) { var blockObj *types.Block @@ -132,7 +135,7 @@ func (consensus *Consensus) validateNewBlock(recvMsg *FBFTMessage) (*types.Block return nil, errors.New("nil block verifier") } - if err := consensus.VerifyBlock(&blockObj); err != nil { + if err := consensus.verifyBlock(&blockObj); err != nil { consensus.getLogger().Error().Err(err).Msg("[validateNewBlock] Block verification failed") return nil, errors.New("Block verification failed") } diff --git a/consensus/view_change_msg.go b/consensus/view_change_msg.go index be19741055..2433abdcbe 100644 --- a/consensus/view_change_msg.go +++ b/consensus/view_change_msg.go @@ -45,7 +45,7 @@ func (consensus *Consensus) constructViewChangeMessage(priKey *bls.PrivateKeyWra Interface("preparedMsg", preparedMsg). Msg("[constructViewChangeMessage] found prepared msg") if block != nil { - if err := consensus.VerifyBlock(block); err == nil { + if err := consensus.verifyBlock(block); err == nil { tmpEncoded, err := rlp.EncodeToBytes(block) if err != nil { consensus.getLogger().Err(err).Msg("[constructViewChangeMessage] Failed encoding block") From 96288e7e08309d71a4882daf8601b39a3f49e90d Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sun, 6 Nov 2022 23:55:22 +0700 Subject: [PATCH 088/420] Make IsLeader private. --- consensus/consensus_service.go | 11 ++++++++++- consensus/consensus_v2.go | 10 +++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index bf420177cc..fdbcbc0457 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -408,7 +408,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // If the leader changed and I myself become the leader if (oldLeader != nil && consensus.LeaderPubKey != nil && - !consensus.LeaderPubKey.Object.IsEqual(oldLeader.Object)) && consensus.IsLeader() { + !consensus.LeaderPubKey.Object.IsEqual(oldLeader.Object)) && consensus.isLeader() { go func() { consensus.getLogger().Info(). Str("myKey", myPubKeys.SerializeToHexStr()). @@ -429,6 +429,15 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() + + return consensus.isLeader() +} + +// IsLeader check if the node is a leader or not by comparing the public key of +// the node with the leader public key +func (consensus *Consensus) isLeader() bool { obj := consensus.LeaderPubKey.Object for _, key := range consensus.priKey { if key.Pub.Object.IsEqual(obj) { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 18feeae286..1fc26e7607 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -100,8 +100,8 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb canHandleViewChange := true intendedForValidator, intendedForLeader := - !consensus.IsLeader(), - consensus.IsLeader() + !consensus.isLeader(), + consensus.isLeader() // if in backup normal mode, force ignore view change event and leader event. if consensus.current.Mode() == NormalBackup { @@ -183,7 +183,7 @@ func (consensus *Consensus) finalCommit() { // Note: leader already sent 67% commit in preCommit. The 100% commit won't be sent immediately // to save network traffic. It will only be sent in retry if consensus doesn't move forward. // Or if the leader is changed for next block, the 100% committed sig will be sent to the next leader immediately. - if !consensus.IsLeader() || block.IsLastBlockInEpoch() { + if !consensus.isLeader() || block.IsLastBlockInEpoch() { // send immediately if err := consensus.msgSender.SendWithRetry( block.NumberU64(), @@ -248,7 +248,7 @@ func (consensus *Consensus) finalCommit() { // If still the leader, send commit sig/bitmap to finish the new block proposal, // else, the block proposal will timeout by itself. - if consensus.IsLeader() { + if consensus.isLeader() { if block.IsLastBlockInEpoch() { // No pipelining go func() { @@ -330,7 +330,7 @@ func (consensus *Consensus) Start( func (consensus *Consensus) StartChannel() { consensus.mutex.Lock() - consensus.isInitialLeader = consensus.IsLeader() + consensus.isInitialLeader = consensus.isLeader() if consensus.isInitialLeader { consensus.start = true consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Send ReadySignal") From ae37eb952da2720a16216684165719bb5af2cdff Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 00:14:19 +0700 Subject: [PATCH 089/420] Make ParseFBFTMessage private. --- consensus/consensus_v2.go | 2 +- consensus/construct.go | 2 +- consensus/fbft_log.go | 6 ++++++ consensus/validator.go | 6 +----- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 1fc26e7607..77beac988c 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -92,7 +92,7 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb members := consensus.Decider.Participants() fbftMsg, err = ParseNewViewMessage(msg, members) default: - fbftMsg, err = consensus.ParseFBFTMessage(msg) + fbftMsg, err = consensus.parseFBFTMessage(msg) } if err != nil || fbftMsg == nil { return errors.Wrapf(err, "unable to parse consensus msg with type: %s", msg.Type) diff --git a/consensus/construct.go b/consensus/construct.go index bbee71203a..8d9f3838aa 100644 --- a/consensus/construct.go +++ b/consensus/construct.go @@ -143,7 +143,7 @@ func (consensus *Consensus) construct( return nil, err } - FBFTMsg, err2 := consensus.ParseFBFTMessage(message) + FBFTMsg, err2 := consensus.parseFBFTMessage(message) if err2 != nil { utils.Logger().Error().Err(err). diff --git a/consensus/fbft_log.go b/consensus/fbft_log.go index 4743085ab6..ba37451038 100644 --- a/consensus/fbft_log.go +++ b/consensus/fbft_log.go @@ -287,6 +287,12 @@ func (log *FBFTLog) FindMessageByMaxViewID(msgs []*FBFTMessage) *FBFTMessage { // ParseFBFTMessage parses FBFT message into FBFTMessage structure func (consensus *Consensus) ParseFBFTMessage(msg *msg_pb.Message) (*FBFTMessage, error) { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + return consensus.parseFBFTMessage(msg) +} + +func (consensus *Consensus) parseFBFTMessage(msg *msg_pb.Message) (*FBFTMessage, error) { // TODO Have this do sanity checks on the message please pbftMsg := FBFTMessage{} pbftMsg.MessageType = msg.GetType() diff --git a/consensus/validator.go b/consensus/validator.go index b33ac6d688..cb2971e4cf 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -18,8 +18,7 @@ import ( ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { - - recvMsg, err := consensus.ParseFBFTMessage(msg) + recvMsg, err := consensus.parseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). Err(err). @@ -27,9 +26,6 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Msg("[OnAnnounce] Unparseable leader message") return } - if consensus.ShardID == 0 { - //fmt.Println("onAnnounce called ", recvMsg.BlockNum) - } // NOTE let it handle its own logs if !consensus.onAnnounceSanityChecks(recvMsg) { From f63b6908f8def96de048f257c9e842bb2b090ac5 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 11:46:49 +0700 Subject: [PATCH 090/420] Fix remove locks. --- consensus/consensus_service.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index fdbcbc0457..4784b99fc0 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -429,8 +429,9 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { - consensus.mutex.RLock() - defer consensus.mutex.RUnlock() + // TODO: if remove locks blockchain stucks. + //consensus.mutex.RLock() + //defer consensus.mutex.RUnlock() return consensus.isLeader() } From 8b5493c82d6f24c6aec908957acf2281b02c1ba6 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 11:58:01 +0700 Subject: [PATCH 091/420] Added additional locks. --- consensus/consensus_v2.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 77beac988c..a862d725ab 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -436,7 +436,8 @@ func (consensus *Consensus) BlockChannel(newBlock *types.Block) { consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") time.AfterFunc(time.Until(consensus.NextBlockDue), func() { consensus.StartFinalityCount() - + consensus.mutex.Lock() + defer consensus.mutex.Unlock() // Update time due for next block consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) From d5015e39970744b8b702d14ddab12b78b59ba582 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 12:05:41 +0700 Subject: [PATCH 092/420] Added additional locks. --- consensus/validator.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/consensus/validator.go b/consensus/validator.go index cb2971e4cf..79e988fce4 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -271,6 +271,8 @@ func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { return } curBlockNum := consensus.BlockNum() + consensus.mutex.Lock() + defer consensus.mutex.Unlock() for _, committedMsg := range consensus.FBFTLog.GetNotVerifiedCommittedMessages(blockObj.NumberU64(), blockObj.Header().ViewID().Uint64(), blockObj.Hash()) { if committedMsg != nil { consensus.onCommitted(committedMsg) @@ -284,9 +286,6 @@ func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - consensus.getLogger().Info(). Uint64("MsgBlockNum", recvMsg.BlockNum). Uint64("MsgViewID", recvMsg.ViewID). From e01c81401933b9fee07cfdf2517d480be2f49870 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 12:22:26 +0700 Subject: [PATCH 093/420] Added readSignatureBitmapPayload locks. --- consensus/consensus_service.go | 15 ++++++++++----- consensus/validator.go | 4 ++-- consensus/view_change.go | 2 +- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 4784b99fc0..53a1a1a7c0 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -7,6 +7,7 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/crypto/bls" + "github.com/harmony-one/harmony/multibls" "github.com/ethereum/go-ethereum/common" protobuf "github.com/golang/protobuf/proto" @@ -229,16 +230,20 @@ func (consensus *Consensus) SetBlockNum(blockNum uint64) { } // ReadSignatureBitmapPayload read the payload for signature and bitmap; offset is the beginning position of reading -func (consensus *Consensus) ReadSignatureBitmapPayload( - recvPayload []byte, offset int, -) (*bls_core.Sign, *bls_cosi.Mask, error) { +func (consensus *Consensus) ReadSignatureBitmapPayload(recvPayload []byte, offset int) (*bls_core.Sign, *bls_cosi.Mask, error) { + consensus.mutex.RLock() + members := consensus.Decider.Participants() + consensus.mutex.RUnlock() + return consensus.readSignatureBitmapPayload(recvPayload, offset, members) +} + +func (consensus *Consensus) readSignatureBitmapPayload(recvPayload []byte, offset int, members multibls.PublicKeys) (*bls_core.Sign, *bls_cosi.Mask, error) { if offset+bls.BLSSignatureSizeInBytes > len(recvPayload) { return nil, nil, errors.New("payload not have enough length") } sigAndBitmapPayload := recvPayload[offset:] // TODO(audit): keep a Mask in the Decider so it won't be reconstructed on the fly. - members := consensus.Decider.Participants() return chain.ReadSignatureBitmapByPublicKeys( sigAndBitmapPayload, members, ) @@ -524,7 +529,7 @@ func (consensus *Consensus) selfCommit(payload []byte) error { return errGetPreparedBlock } - aggSig, mask, err := consensus.ReadSignatureBitmapPayload(payload, 32) + aggSig, mask, err := consensus.readSignatureBitmapPayload(payload, 32, consensus.Decider.Participants()) if err != nil { return errReadBitmapPayload } diff --git a/consensus/validator.go b/consensus/validator.go index 79e988fce4..7336c7b477 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -62,7 +62,7 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { if len(recvMsg.Block) > 0 { go func() { // Best effort check, no need to error out. - _, err := consensus.validateNewBlock(recvMsg) + _, err := consensus.ValidateNewBlock(recvMsg) if err == nil { consensus.getLogger().Info(). @@ -205,7 +205,7 @@ func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { // check validity of prepared signature blockHash := recvMsg.BlockHash - aggSig, mask, err := consensus.ReadSignatureBitmapPayload(recvMsg.Payload, 0) + aggSig, mask, err := consensus.readSignatureBitmapPayload(recvMsg.Payload, 0, consensus.Decider.Participants()) if err != nil { consensus.getLogger().Error().Err(err).Msg("ReadSignatureBitmapPayload failed!") return diff --git a/consensus/view_change.go b/consensus/view_change.go index 95ccbb97aa..9951cf06ba 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -501,7 +501,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { utils.CountOneBits(m3Mask.Bitmap) > utils.CountOneBits(m2Mask.Bitmap)) { // m1 is not empty, check it's valid blockHash := recvMsg.Payload[:32] - aggSig, mask, err := consensus.ReadSignatureBitmapPayload(recvMsg.Payload, 32) + aggSig, mask, err := consensus.readSignatureBitmapPayload(recvMsg.Payload, 32, consensus.Decider.Participants()) if err != nil { consensus.getLogger().Error().Err(err). Msg("[onNewView] ReadSignatureBitmapPayload Failed") From b9af100719c85b27842c2cf8691798de487ab460 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 12:50:03 +0700 Subject: [PATCH 094/420] Added HandleMessageUpdate locks. --- consensus/consensus.go | 8 ++++---- consensus/consensus_service.go | 7 ++++++- consensus/consensus_v2.go | 8 +++++--- consensus/leader.go | 9 +-------- consensus/threshold.go | 2 +- consensus/view_change.go | 11 ++--------- 6 files changed, 19 insertions(+), 26 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 887b4744e9..11a0c29c89 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -200,7 +200,7 @@ func (consensus *Consensus) GetPrivateKeys() multibls.PrivateKeys { } // GetLeaderPrivateKey returns leader private key if node is the leader -func (consensus *Consensus) GetLeaderPrivateKey(leaderKey *bls_core.PublicKey) (*bls.PrivateKeyWrapper, error) { +func (consensus *Consensus) getLeaderPrivateKey(leaderKey *bls_core.PublicKey) (*bls.PrivateKeyWrapper, error) { for i, key := range consensus.priKey { if key.Pub.Object.IsEqual(leaderKey) { return &consensus.priKey[i], nil @@ -209,9 +209,9 @@ func (consensus *Consensus) GetLeaderPrivateKey(leaderKey *bls_core.PublicKey) ( return nil, errors.Wrapf(errLeaderPriKeyNotFound, leaderKey.SerializeToHexStr()) } -// GetConsensusLeaderPrivateKey returns consensus leader private key if node is the leader -func (consensus *Consensus) GetConsensusLeaderPrivateKey() (*bls.PrivateKeyWrapper, error) { - return consensus.GetLeaderPrivateKey(consensus.LeaderPubKey.Object) +// getConsensusLeaderPrivateKey returns consensus leader private key if node is the leader +func (consensus *Consensus) getConsensusLeaderPrivateKey() (*bls.PrivateKeyWrapper, error) { + return consensus.getLeaderPrivateKey(consensus.LeaderPubKey.Object) } // SetBlockVerifier sets the block verifier diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 53a1a1a7c0..d613f70004 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -468,7 +468,12 @@ func (consensus *Consensus) isLeader() bool { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { - fmt.Println("SetViewIDs", height) + consensus.setViewIDs(height) +} + +// SetViewIDs set both current view ID and view changing ID to the height +// of the blockchain. It is used during client startup to recover the state +func (consensus *Consensus) setViewIDs(height uint64) { consensus.SetCurBlockViewID(height) consensus.SetViewChangingID(height) } diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index a862d725ab..e866e1af71 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -56,6 +56,8 @@ func (consensus *Consensus) isViewChangingMode() bool { // HandleMessageUpdate will update the consensus state according to received message func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb.Message, senderKey *bls.SerializedPublicKey) error { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() // when node is in ViewChanging mode, it still accepts normal messages into FBFTLog // in order to avoid possible trap forever but drop PREPARE and COMMIT // which are message types specifically for a node acting as leader @@ -144,7 +146,7 @@ func (consensus *Consensus) finalCommit() { Msg("[finalCommit] Finalizing Consensus") beforeCatchupNum := consensus.BlockNum() - leaderPriKey, err := consensus.GetConsensusLeaderPrivateKey() + leaderPriKey, err := consensus.getConsensusLeaderPrivateKey() if err != nil { consensus.getLogger().Error().Err(err).Msg("[finalCommit] leader not found") return @@ -559,7 +561,7 @@ func (consensus *Consensus) preCommitAndPropose(blk *types.Block) error { return errors.New("block to pre-commit is nil") } - leaderPriKey, err := consensus.GetConsensusLeaderPrivateKey() + leaderPriKey, err := consensus.getConsensusLeaderPrivateKey() if err != nil { consensus.getLogger().Error().Err(err).Msg("[preCommitAndPropose] leader not found") return err @@ -804,7 +806,7 @@ func (consensus *Consensus) postCatchup(initBN uint64) { // GenerateVrfAndProof generates new VRF/Proof from hash of previous block func (consensus *Consensus) GenerateVrfAndProof(newHeader *block.Header) error { - key, err := consensus.GetConsensusLeaderPrivateKey() + key, err := consensus.getConsensusLeaderPrivateKey() if err != nil { return errors.New("[GenerateVrfAndProof] no leader private key provided") } diff --git a/consensus/leader.go b/consensus/leader.go index 6b176d27ae..548ec98c0a 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -29,7 +29,7 @@ func (consensus *Consensus) announce(block *types.Block) { copy(consensus.blockHash[:], blockHash[:]) consensus.block = encodedBlock // Must set block bytes before consensus.construct() - key, err := consensus.GetConsensusLeaderPrivateKey() + key, err := consensus.getConsensusLeaderPrivateKey() if err != nil { consensus.getLogger().Warn().Err(err).Msg("[Announce] Node not a leader") return @@ -102,11 +102,6 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { Uint64("MsgBlockNum", recvMsg.BlockNum). Msg("[OnPrepare] No Matching Announce message") } - - //// Read - Start - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - if !consensus.isRightBlockNumAndViewID(recvMsg) { return } @@ -195,8 +190,6 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { - consensus.mutex.Lock() - defer consensus.mutex.Unlock() //// Read - Start if !consensus.isRightBlockNumAndViewID(recvMsg) { return diff --git a/consensus/threshold.go b/consensus/threshold.go index e8bd875cba..ecb54980a0 100644 --- a/consensus/threshold.go +++ b/consensus/threshold.go @@ -15,7 +15,7 @@ import ( func (consensus *Consensus) didReachPrepareQuorum() error { logger := utils.Logger() logger.Info().Msg("[OnPrepare] Received Enough Prepare Signatures") - leaderPriKey, err := consensus.GetConsensusLeaderPrivateKey() + leaderPriKey, err := consensus.getConsensusLeaderPrivateKey() if err != nil { utils.Logger().Warn().Err(err).Msg("[OnPrepare] leader not found") return err diff --git a/consensus/view_change.go b/consensus/view_change.go index 9951cf06ba..720205ee83 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -347,10 +347,6 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri // onViewChange is called when the view change message is received. func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { - //fmt.Printf("[onViewChange] received view change message from %+v\n", recvMsg) - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - consensus.getLogger().Debug(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -359,7 +355,7 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { // if not leader, noop newLeaderKey := recvMsg.LeaderPubkey - newLeaderPriKey, err := consensus.GetLeaderPrivateKey(newLeaderKey.Object) + newLeaderPriKey, err := consensus.getLeaderPrivateKey(newLeaderKey.Object) if err != nil { consensus.getLogger().Debug(). Err(err). @@ -454,9 +450,6 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { // Or the validator will enter announce phase to wait for the new block proposed // from the new leader func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - consensus.getLogger().Info(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -543,7 +536,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.consensusTimeout[timeoutViewChange].Stop() // newView message verified success, override my state - consensus.SetViewIDs(recvMsg.ViewID) + consensus.setViewIDs(recvMsg.ViewID) consensus.LeaderPubKey = senderKey consensus.ResetViewChangeState() From 3b49a6b7fef2d99aeda8f13dfba3fc630c89f567 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 13:08:15 +0700 Subject: [PATCH 095/420] Added LastMile locks. --- api/service/legacysync/syncing.go | 26 +++++++++++++------------- consensus/consensus_v2.go | 10 +++++----- consensus/downloader.go | 29 ++++++++++++++--------------- 3 files changed, 32 insertions(+), 33 deletions(-) diff --git a/api/service/legacysync/syncing.go b/api/service/legacysync/syncing.go index 43f07c039b..3375ccdc50 100644 --- a/api/service/legacysync/syncing.go +++ b/api/service/legacysync/syncing.go @@ -18,6 +18,7 @@ import ( "github.com/harmony-one/harmony/api/service/legacysync/downloader" pb "github.com/harmony-one/harmony/api/service/legacysync/downloader/proto" "github.com/harmony-one/harmony/consensus" + consensus2 "github.com/harmony-one/harmony/consensus" "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" @@ -1142,20 +1143,19 @@ func (ss *StateSync) SyncLoop(bc core.BlockChain, worker *worker.Worker, isBeaco func (ss *StateSync) addConsensusLastMile(bc core.BlockChain, consensus *consensus.Consensus) error { curNumber := bc.CurrentBlock().NumberU64() - blockIter, err := consensus.GetLastMileBlockIter(curNumber + 1) - if err != nil { - return err - } - for { - block := blockIter.Next() - if block == nil { - break - } - if _, err := bc.InsertChain(types.Blocks{block}, true); err != nil { - return errors.Wrap(err, "failed to InsertChain") + err := consensus.GetLastMileBlockIter(curNumber+1, func(blockIter *consensus2.LastMileBlockIter) error { + for { + block := blockIter.Next() + if block == nil { + break + } + if _, err := bc.InsertChain(types.Blocks{block}, true); err != nil { + return errors.Wrap(err, "failed to InsertChain") + } } - } - return nil + return nil + }) + return err } // GetSyncingPort returns the syncing port. diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index e866e1af71..4253a46e95 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -496,24 +496,24 @@ type LastMileBlockIter struct { } // GetLastMileBlockIter get the iterator of the last mile blocks starting from number bnStart -func (consensus *Consensus) GetLastMileBlockIter(bnStart uint64) (*LastMileBlockIter, error) { +func (consensus *Consensus) GetLastMileBlockIter(bnStart uint64, cb func(iter *LastMileBlockIter) error) error { consensus.mutex.Lock() defer consensus.mutex.Unlock() if consensus.BlockVerifier == nil { - return nil, errors.New("consensus haven't initialized yet") + return errors.New("consensus haven't initialized yet") } blocks, _, err := consensus.getLastMileBlocksAndMsg(bnStart) if err != nil { - return nil, err + return err } - return &LastMileBlockIter{ + return cb(&LastMileBlockIter{ blockCandidates: blocks, fbftLog: consensus.FBFTLog, verify: consensus.BlockVerifier, curIndex: 0, logger: consensus.getLogger(), - }, nil + }) } // Next iterate to the next last mile block diff --git a/consensus/downloader.go b/consensus/downloader.go index 2734f8baed..8442ed5343 100644 --- a/consensus/downloader.go +++ b/consensus/downloader.go @@ -76,7 +76,7 @@ func (dh *downloadHelper) downloadFinishedLoop() { for { select { case <-dh.finishedCh: - err := dh.c.addConsensusLastMile() + err := dh.c.AddConsensusLastMile() if err != nil { dh.c.getLogger().Error().Err(err).Msg("add last mile failed") } @@ -89,22 +89,21 @@ func (dh *downloadHelper) downloadFinishedLoop() { } } -func (consensus *Consensus) addConsensusLastMile() error { +func (consensus *Consensus) AddConsensusLastMile() error { curBN := consensus.Blockchain().CurrentBlock().NumberU64() - blockIter, err := consensus.GetLastMileBlockIter(curBN + 1) - if err != nil { - return err - } - for { - block := blockIter.Next() - if block == nil { - break - } - if _, err := consensus.Blockchain().InsertChain(types.Blocks{block}, true); err != nil { - return errors.Wrap(err, "failed to InsertChain") + err := consensus.GetLastMileBlockIter(curBN+1, func(blockIter *LastMileBlockIter) error { + for { + block := blockIter.Next() + if block == nil { + break + } + if _, err := consensus.Blockchain().InsertChain(types.Blocks{block}, true); err != nil { + return errors.Wrap(err, "failed to InsertChain") + } } - } - return nil + return nil + }) + return err } func (consensus *Consensus) spinUpStateSync() { From c7cdf758bbc48aed63782f7cb75e797c8be0c9d8 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 17:23:59 +0700 Subject: [PATCH 096/420] Locks for IsValidatorInCommittee. --- consensus/consensus_service.go | 6 ++++++ consensus/validator.go | 2 +- consensus/view_change.go | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index d613f70004..fdf0880625 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -159,6 +159,12 @@ func (consensus *Consensus) resetState() { // IsValidatorInCommittee returns whether the given validator BLS address is part of my committee func (consensus *Consensus) IsValidatorInCommittee(pubKey bls.SerializedPublicKey) bool { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() + return consensus.isValidatorInCommittee(pubKey) +} + +func (consensus *Consensus) isValidatorInCommittee(pubKey bls.SerializedPublicKey) bool { return consensus.Decider.IndexOf(pubKey) != -1 } diff --git a/consensus/validator.go b/consensus/validator.go index 7336c7b477..2ccc453645 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -408,7 +408,7 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { func (consensus *Consensus) getPriKeysInCommittee() []*bls.PrivateKeyWrapper { priKeys := []*bls.PrivateKeyWrapper{} for i, key := range consensus.priKey { - if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { + if !consensus.isValidatorInCommittee(key.Pub.Bytes) { continue } priKeys = append(priKeys, &consensus.priKey[i]) diff --git a/consensus/view_change.go b/consensus/view_change.go index 720205ee83..1a2accf727 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -279,7 +279,7 @@ func (consensus *Consensus) startViewChange() { // for view change, send separate view change per public key // do not do multi-sign of view change message for _, key := range consensus.priKey { - if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { + if !consensus.isValidatorInCommittee(key.Pub.Bytes) { continue } msgToSend := consensus.constructViewChangeMessage(&key) From e4f56eb56148692388b72091fc9e2830dfe57d9e Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 20:29:44 +0700 Subject: [PATCH 097/420] Fixed locks. --- consensus/consensus_msg_sender.go | 10 ++++++++-- consensus/consensus_service.go | 11 ++++++----- consensus/consensus_v2.go | 9 ++++++--- node/node_handler.go | 5 ++--- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/consensus/consensus_msg_sender.go b/consensus/consensus_msg_sender.go index 003d89cd96..ffa9d8b1c6 100644 --- a/consensus/consensus_msg_sender.go +++ b/consensus/consensus_msg_sender.go @@ -67,7 +67,10 @@ func (sender *MessageSender) SendWithRetry(blockNum uint64, msgType msg_pb.Messa sender.Retry(&msgRetry) }() } - return sender.host.SendMessageToGroups(groups, p2pMsg) + // MessageSender lays inside consensus, but internally calls consensus public api. + // Tt would be deadlock if run in current thread. + go sender.host.SendMessageToGroups(groups, p2pMsg) + return nil } // DelayedSendWithRetry is similar to SendWithRetry but without the initial message sending but only retries. @@ -86,7 +89,10 @@ func (sender *MessageSender) DelayedSendWithRetry(blockNum uint64, msgType msg_p // SendWithoutRetry sends message without retry logic. func (sender *MessageSender) SendWithoutRetry(groups []nodeconfig.GroupID, p2pMsg []byte) error { - return sender.host.SendMessageToGroups(groups, p2pMsg) + // MessageSender lays inside consensus, but internally calls consensus public api. + // It would be deadlock if run in current thread. + go sender.host.SendMessageToGroups(groups, p2pMsg) + return nil } // Retry will retry the consensus message for times. diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index fdf0880625..daaa219842 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -74,6 +74,8 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) consensus.getLogger().Info().Msg("My Committee updated") for i := range pubKeys { @@ -94,7 +96,7 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi } // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. - consensus.UpdateBitmaps() + consensus.updateBitmaps() consensus.resetState() // do not reset view change state if it is in view changing mode @@ -126,7 +128,7 @@ func (consensus *Consensus) signConsensusMessage(message *msg_pb.Message, } // UpdateBitmaps update the bitmaps for prepare and commit phase -func (consensus *Consensus) UpdateBitmaps() { +func (consensus *Consensus) updateBitmaps() { consensus.getLogger().Debug(). Str("MessageType", consensus.phase.String()). Msg("[UpdateBitmaps] Updating consensus bitmaps") @@ -440,9 +442,8 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { - // TODO: if remove locks blockchain stucks. - //consensus.mutex.RLock() - //defer consensus.mutex.RUnlock() + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() return consensus.isLeader() } diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 4253a46e95..37adeea430 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -703,11 +703,14 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess } consensus.FinishFinalityCount() - consensus.PostConsensusJob(blk) - consensus.SetupForNewConsensus(blk, committedMsg) + go func() { + consensus.PostConsensusJob(blk) + }() + consensus.setupForNewConsensus(blk, committedMsg) utils.Logger().Info().Uint64("blockNum", blk.NumberU64()). Str("hash", blk.Header().Hash().Hex()). Msg("Added New Block to Blockchain!!!") + return nil } @@ -756,7 +759,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { } // SetupForNewConsensus sets the state for new consensus -func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { +func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) consensus.SetCurBlockViewID(committedMsg.ViewID + 1) consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] diff --git a/node/node_handler.go b/node/node_handler.go index d15b41eb5a..3589d19660 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -361,9 +361,8 @@ func VerifyNewBlock(nodeConfig *nodeconfig.ConfigType, blockChain core.BlockChai } // PostConsensusProcessing is called by consensus participants, after consensus is done, to: -// 1. add the new block to blockchain -// 2. [leader] send new block to the client -// 3. [leader] send cross shard tx receipts to destination shard +// 1. [leader] send new block to the client +// 2. [leader] send cross shard tx receipts to destination shard func (node *Node) PostConsensusProcessing(newBlock *types.Block) error { if node.Consensus.IsLeader() { if node.IsRunningBeaconChain() { From 9bac4b857073bcaafcb35f7e3df996c8e9220dec Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 20:44:43 +0700 Subject: [PATCH 098/420] Fixed tests. --- consensus/construct_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/construct_test.go b/consensus/construct_test.go index 71da42e3f4..1add32219d 100644 --- a/consensus/construct_test.go +++ b/consensus/construct_test.go @@ -67,8 +67,8 @@ func TestConstructPreparedMessage(test *testing.T) { if err != nil { test.Fatalf("Cannot craeate consensus: %v", err) } - consensus.ResetState() - consensus.UpdateBitmaps() + consensus.resetState() + consensus.updateBitmaps() consensus.blockHash = [32]byte{} message := "test string" From 7ad82d290e16af303fbed8e6ca6239869127b460 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 8 Nov 2022 01:05:23 +0700 Subject: [PATCH 099/420] Fixed tests. --- consensus/consensus_service.go | 6 +++++- internal/utils/singleton.go | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index daaa219842..e54f5edf21 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -442,7 +442,11 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { - consensus.mutex.RLock() + _ = utils.AssertNoLongerThan0(5*time.Second, func() error { + consensus.mutex.RLock() + return nil + }) + defer consensus.mutex.RUnlock() return consensus.isLeader() diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index 10101d7673..e58a0cc96d 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path" + "runtime/debug" "strconv" "sync" "time" @@ -212,3 +213,19 @@ func GetPort() int { } return 0 } + +func AssertNoLongerThan0[E any](t time.Duration, f func() E) E { + ch := make(chan E) + defer close(ch) + stack := debug.Stack() + go func() { + select { + case <-time.After(t): + panic("AssertNoLongerThan0: " + string(stack)) + case <-ch: + return + } + }() + + return f() +} From 24c31627fa777029cf5bc4426e39c72c8bb95282 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 8 Nov 2022 13:34:53 +0700 Subject: [PATCH 100/420] Fixed lock. --- consensus/consensus_service.go | 37 +++++++++++++++++++++++++++++----- consensus/consensus_v2.go | 10 ++++----- internal/utils/singleton.go | 16 +++++++++++++++ 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e54f5edf21..e539264dbe 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -76,6 +76,10 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { consensus.mutex.Lock() defer consensus.mutex.Unlock() + return consensus.updatePublicKeys(pubKeys, allowlist) +} + +func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { consensus.Decider.UpdateParticipants(pubKeys, allowlist) consensus.getLogger().Info().Msg("My Committee updated") for i := range pubKeys { @@ -172,6 +176,13 @@ func (consensus *Consensus) isValidatorInCommittee(pubKey bls.SerializedPublicKe // SetMode sets the mode of consensus func (consensus *Consensus) SetMode(m Mode) { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + consensus.setMode(m) +} + +// SetMode sets the mode of consensus +func (consensus *Consensus) setMode(m Mode) { if m == Normal && consensus.isBackup { m = NormalBackup } @@ -212,8 +223,8 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if consensus.IgnoreViewIDCheck.IsSet() { //in syncing mode, node accepts incoming messages without viewID/leaderKey checking //so only set mode to normal when new node enters consensus and need checking viewID - consensus.SetMode(Normal) - consensus.SetViewIDs(msg.ViewID) + consensus.setMode(Normal) + consensus.setViewIDs(msg.ViewID) if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } @@ -268,6 +279,12 @@ func (consensus *Consensus) readSignatureBitmapPayload(recvPayload []byte, offse // (b) node in committed but has any err during processing: Syncing mode // (c) node in committed and everything looks good: Normal mode func (consensus *Consensus) UpdateConsensusInformation() Mode { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + return consensus.updateConsensusInformation() +} + +func (consensus *Consensus) updateConsensusInformation() Mode { curHeader := consensus.Blockchain().CurrentHeader() curEpoch := curHeader.Epoch() nextEpoch := new(big.Int).Add(curHeader.Epoch(), common.Big1) @@ -372,7 +389,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { consensus.getLogger().Info(). Int("numPubKeys", len(pubKeys)). Msg("[UpdateConsensusInformation] Successfully updated public keys") - consensus.UpdatePublicKeys(pubKeys, shard.Schedule.InstanceForEpoch(nextEpoch).ExternalAllowlist()) + consensus.updatePublicKeys(pubKeys, shard.Schedule.InstanceForEpoch(nextEpoch).ExternalAllowlist()) // Update voters in the committee if _, err := consensus.Decider.SetVoters( @@ -485,8 +502,8 @@ func (consensus *Consensus) SetViewIDs(height uint64) { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) setViewIDs(height uint64) { - consensus.SetCurBlockViewID(height) - consensus.SetViewChangingID(height) + consensus.setCurBlockViewID(height) + consensus.setViewChangingID(height) } // SetCurBlockViewID set the current view ID @@ -494,11 +511,21 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } +// SetCurBlockViewID set the current view ID +func (consensus *Consensus) setCurBlockViewID(viewID uint64) { + consensus.current.SetCurBlockViewID(viewID) +} + // SetViewChangingID set the current view change ID func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) } +// SetViewChangingID set the current view change ID +func (consensus *Consensus) setViewChangingID(viewID uint64) { + consensus.current.SetViewChangingID(viewID) +} + // StartFinalityCount set the finality counter to current time func (consensus *Consensus) StartFinalityCount() { consensus.finalityCounter.Store(time.Now().UnixNano()) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 37adeea430..a30bc392a9 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -348,7 +348,7 @@ func (consensus *Consensus) syncReadyChan() { if consensus.BlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) consensus.SetViewIDs(consensus.Blockchain().CurrentHeader().ViewID().Uint64() + 1) - mode := consensus.UpdateConsensusInformation() + mode := consensus.updateConsensusInformation() consensus.current.SetMode(mode) consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") consensus.consensusTimeout[timeoutConsensus].Start() @@ -357,8 +357,8 @@ func (consensus *Consensus) syncReadyChan() { } else if consensus.Mode() == Syncing { // Corner case where sync is triggered before `onCommitted` and there is a race // for block insertion between consensus and downloader. - mode := consensus.UpdateConsensusInformation() - consensus.SetMode(mode) + mode := consensus.updateConsensusInformation() + consensus.setMode(mode) consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") consensus.consensusTimeout[timeoutConsensus].Start() consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() @@ -761,7 +761,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { // SetupForNewConsensus sets the state for new consensus func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) - consensus.SetCurBlockViewID(committedMsg.ViewID + 1) + consensus.setCurBlockViewID(committedMsg.ViewID + 1) consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] var epoch *big.Int if blk.IsLastBlockInEpoch() { @@ -775,7 +775,7 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg // Update consensus keys at last so the change of leader status doesn't mess up normal flow if blk.IsLastBlockInEpoch() { - consensus.SetMode(consensus.UpdateConsensusInformation()) + consensus.setMode(consensus.updateConsensusInformation()) } consensus.FBFTLog.PruneCacheBeforeBlock(blk.NumberU64()) consensus.resetState() diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index e58a0cc96d..b940fbfeb4 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -229,3 +229,19 @@ func AssertNoLongerThan0[E any](t time.Duration, f func() E) E { return f() } + +func AssertNoLongerThan(t time.Duration, f func()) { + ch := make(chan struct{}) + defer close(ch) + stack := debug.Stack() + go func() { + select { + case <-time.After(t): + panic("AssertNoLongerThan0: " + string(stack)) + case <-ch: + return + } + }() + + f() +} From 93ef7fe0347fafab045c3439851090e83d5c1d7b Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 8 Nov 2022 19:30:57 +0700 Subject: [PATCH 101/420] Rebased over leader rotation. --- consensus/consensus.go | 8 +++++--- consensus/consensus_service.go | 18 +----------------- consensus/consensus_v2.go | 18 ++++-------------- consensus/view_change.go | 2 +- 4 files changed, 11 insertions(+), 35 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 11a0c29c89..10144c9108 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -178,6 +178,10 @@ func (consensus *Consensus) getPublicKeys() multibls.PublicKeys { } func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper { + return consensus.getLeaderPubKey() +} + +func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } @@ -185,10 +189,8 @@ func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } -func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { - consensus.pubKeyLock.Lock() +func (consensus *Consensus) setLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { consensus.LeaderPubKey = pub - consensus.pubKeyLock.Unlock() } func (consensus *Consensus) setLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e539264dbe..1062b4502a 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -459,11 +459,7 @@ func (consensus *Consensus) updateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { - _ = utils.AssertNoLongerThan0(5*time.Second, func() error { - consensus.mutex.RLock() - return nil - }) - + consensus.mutex.RLock() defer consensus.mutex.RUnlock() return consensus.isLeader() @@ -481,18 +477,6 @@ func (consensus *Consensus) isLeader() bool { return false } -// isLeader check if the node is a leader or not by comparing the public key of -// the node with the leader public key. This function assume it runs under lock. -func (consensus *Consensus) isLeader() bool { - obj := consensus.LeaderPubKey.Object - for _, key := range consensus.priKey { - if key.Pub.Object.IsEqual(obj) { - return true - } - } - return false -} - // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index a30bc392a9..b8696200c3 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -460,16 +460,6 @@ func (consensus *Consensus) BlockChannel(newBlock *types.Block) { } } -// Close closes the consensus. If current is in normal commit phase, wait until the commit -// phase end. -func (consensus *Consensus) Close() error { - if consensus.dHelper != nil { - consensus.dHelper.close() - } - consensus.waitForCommit() - return nil -} - // waitForCommit wait extra 2 seconds for commit phase to finish func (consensus *Consensus) waitForCommit() { if consensus.Mode() != Normal || consensus.phase.Get() != FBFTCommit { @@ -717,11 +707,11 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess // rotateLeader rotates the leader to the next leader in the committee. // This function must be called with enabled leader rotation. func (consensus *Consensus) rotateLeader(epoch *big.Int) { - prev := consensus.GetLeaderPubKey() + prev := consensus.getLeaderPubKey() bc := consensus.Blockchain() curNumber := bc.CurrentHeader().Number().Uint64() utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), bc.Config().IsLeaderRotation(epoch), bc.Config().LeaderRotationBlocksCount) - leader := consensus.GetLeaderPubKey() + leader := consensus.getLeaderPubKey() for i := 0; i < bc.Config().LeaderRotationBlocksCount; i++ { header := bc.GetHeaderByNumber(curNumber - uint64(i)) if header == nil { @@ -748,9 +738,9 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { utils.Logger().Error().Msg("Failed to get next leader") return } else { - consensus.SetLeaderPubKey(next) + consensus.setLeaderPubKey(next) } - if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { + if consensus.isLeader() && !consensus.getLeaderPubKey().Object.IsEqual(prev.Object) { // leader changed go func() { consensus.ReadySignal <- SyncProposal diff --git a/consensus/view_change.go b/consensus/view_change.go index 1a2accf727..d7080c5a79 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -340,7 +340,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.resetState() } - consensus.SetLeaderPubKey(newLeaderPriKey.Pub) + consensus.setLeaderPubKey(newLeaderPriKey.Pub) return nil } From 73bed7f1030f589967a6d6798e0952023cebc297 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Fri, 18 Nov 2022 18:24:48 +0700 Subject: [PATCH 102/420] Fix formatting. --- internal/params/config.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/internal/params/config.go b/internal/params/config.go index f1000098a3..c86d5ce88c 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -319,8 +319,6 @@ var ( big.NewInt(0), // AllowlistEpoch big.NewInt(1), // LeaderRotationEpoch 64, // LeaderRotationBlocksCount - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } @@ -361,9 +359,8 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } From 9bb649422e61ff0bbf041830b8733d328faa9a87 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 17 Jan 2023 12:55:50 -0300 Subject: [PATCH 103/420] Rebased onto dev. --- internal/params/config.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/internal/params/config.go b/internal/params/config.go index c86d5ce88c..725704bd06 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -111,7 +111,6 @@ var ( AllowlistEpoch: big.NewInt(2), LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 64, - TesnetNinetyPercentEpoch: big.NewInt(399), FeeCollectEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. @@ -274,7 +273,6 @@ var ( AllowlistEpoch: EpochTBD, LeaderRotationEpoch: big.NewInt(4), LeaderRotationBlocksCount: 5, - TesnetNinetyPercentEpoch: EpochTBD, FeeCollectEpoch: big.NewInt(5), LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 5, @@ -719,10 +717,6 @@ func (c *ChainConfig) IsLeaderRotation(epoch *big.Int) bool { return isForked(c.LeaderRotationEpoch, epoch) } -func (c *ChainConfig) IsTestnetNinetyPercent(epoch *big.Int) bool { - return isForked(c.TesnetNinetyPercentEpoch, epoch) && c == TestnetChainConfig -} - // IsFeeCollectEpoch determines whether Txn Fees will be collected into the community-managed account. func (c *ChainConfig) IsFeeCollectEpoch(epoch *big.Int) bool { return isForked(c.FeeCollectEpoch, epoch) From 9363d3074493c2397b8e9379aa3445378f215a70 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 1 Sep 2022 00:53:16 +0700 Subject: [PATCH 104/420] in progress. --- api/service/explorer/service.go | 8 +++++--- cmd/harmony/main.go | 3 +++ consensus/consensus.go | 2 ++ consensus/consensus_service.go | 29 +++++++++++++++++++++++++++++ consensus/consensus_v2.go | 17 ++++++++++------- consensus/leader.go | 1 + consensus/quorum/quorum.go | 7 ++++++- consensus/validator.go | 4 ++++ consensus/view_change.go | 8 ++++++++ internal/utils/singleton.go | 33 +-------------------------------- node/node_newblock.go | 2 -- 11 files changed, 69 insertions(+), 45 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 6aa275332b..d96c1bc56c 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -115,6 +115,8 @@ func (s *Service) Run() *http.Server { s.router = mux.NewRouter() + fmt.Println("++", addr) + // Set up router for addresses. // Fetch addresses request, accepts parameter size: how much addresses to read, // parameter prefix: from which address prefix start @@ -221,9 +223,9 @@ func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { for i := cur; i > 0; i-- { block := s.blockchain.GetBlockByNumber(i) - w.Write([]byte(fmt.Sprintf("#%d ", i))) - w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) - w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) + w.Write([]byte(fmt.Sprintf("%d ", i))) + w.Write([]byte(fmt.Sprintf("%s ", block.Header().ViewID().String()))) + w.Write([]byte(fmt.Sprintf("%s ", block.Header().Coinbase().Hash().Hex()))) w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) } } diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 4bb70a81b4..2fcfe5f037 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -266,6 +266,8 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { var err error + fmt.Println("OS: ", os.Args) + nodeconfigSetShardSchedule(hc) nodeconfig.SetShardingSchedule(shard.Schedule) nodeconfig.SetVersion(getHarmonyVersion()) @@ -801,6 +803,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // Set the consensus ID to be the current block number viewID := currentNode.Blockchain().CurrentBlock().Header().ViewID().Uint64() + fmt.Println("viewID:", viewID) currentConsensus.SetViewIDs(viewID + 1) utils.Logger().Info(). Uint64("viewID", viewID). diff --git a/consensus/consensus.go b/consensus/consensus.go index 10144c9108..b73ecfb8e9 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -71,6 +71,8 @@ type Consensus struct { priKey multibls.PrivateKeys // the publickey of leader LeaderPubKey *bls.PublicKeyWrapper + // index of leader in the list of validators. + LeaderIndex int // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 1062b4502a..2fabb42b3c 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync/atomic" "time" @@ -228,6 +229,7 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } + fmt.Println("[checkViewID] Set LEADEER PUB KEY ", msg.SenderPubkeys[0].Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() @@ -477,9 +479,22 @@ func (consensus *Consensus) isLeader() bool { return false } +// isLeader check if the node is a leader or not by comparing the public key of +// the node with the leader public key. This function assume it runs under lock. +func (consensus *Consensus) isLeader() bool { + obj := consensus.LeaderPubKey.Object + for _, key := range consensus.priKey { + if key.Pub.Object.IsEqual(obj) { + return true + } + } + return false +} + // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { + fmt.Println("SetViewIDs", height) consensus.setViewIDs(height) } @@ -495,6 +510,19 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } +// SetLeaderIndex set the leader index. +func (consensus *Consensus) SetLeaderIndex(f func(int) int) (current int) { + consensus.pubKeyLock.Lock() + defer consensus.pubKeyLock.Unlock() + consensus.LeaderIndex = f(consensus.LeaderIndex) + return consensus.LeaderIndex +} + +func (consensus *Consensus) GetLeaderIndex() int { + consensus.pubKeyLock.Lock() + defer consensus.pubKeyLock.Unlock() + return consensus.LeaderIndex +} // SetCurBlockViewID set the current view ID func (consensus *Consensus) setCurBlockViewID(viewID uint64) { consensus.current.SetCurBlockViewID(viewID) @@ -505,6 +533,7 @@ func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) } + // SetViewChangingID set the current view change ID func (consensus *Consensus) setViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index b8696200c3..fea7b7dfe6 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -425,18 +425,20 @@ func (consensus *Consensus) Close() error { } func (consensus *Consensus) BlockChannel(newBlock *types.Block) { - //consensus.ReshardingNextLeader(newBlock)consensus.getLogger().Info(). - Uint64("MsgBlockNum", newBlock.NumberU64()). - Msg("[ConsensusMainLoop] Received Proposed New Block!") + //consensus.ReshardingNextLeader(newBlock) + consensus.getLogger().Info(). + Uint64("MsgBlockNum", newBlock.NumberU64()). + Msg("[ConsensusMainLoop] Received Proposed New Block!") if newBlock.NumberU64() < consensus.BlockNum() { consensus.getLogger().Warn().Uint64("newBlockNum", newBlock.NumberU64()). Msg("[ConsensusMainLoop] received old block, abort") return - } - // Sleep to wait for the full block time - consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") - time.AfterFunc(time.Until(consensus.NextBlockDue), func() { + } + // Sleep to wait for the full block time + consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") + + time.AfterFunc(time.Until(consensus.NextBlockDue), func() { consensus.StartFinalityCount() consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -664,6 +666,7 @@ func (consensus *Consensus) tryCatchup() error { consensus.getLogger().Error().Err(err).Msg("[TryCatchup] Failed to add block to chain") return err } + //fmt.Println("tryCatchup ", utils.GetPort(), blk.NumberU64()) select { // TODO: Remove this when removing dns sync and stream sync is fully up case consensus.VerifiedNewBlock <- blk: diff --git a/consensus/leader.go b/consensus/leader.go index 548ec98c0a..b34757ae64 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -322,4 +322,5 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED) } + //fmt.Println("onCommit99: ", utils.GetPort(), recvMsg.BlockNum) } diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 6cf79e549a..0586ace4fe 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -230,12 +230,17 @@ func (s *cIdentities) NthNextHmy(instance shardingconfig.Instance, pubKey *bls.P Msg("[NthNextHmy] pubKey not found") } numNodes := instance.NumHarmonyOperatedNodesPerShard() + //fmt.Println("??idx:", idx, numNodes) // sanity check to avoid out of bound access if numNodes <= 0 || numNodes > len(s.publicKeys) { numNodes = len(s.publicKeys) } idx = (idx + next) % numNodes - return found, &s.publicKeys[idx] + //fmt.Println("-------idx:", idx) + + new := &s.publicKeys[idx] + fmt.Println("NthNextHmy: ", pubKey.Bytes.Hex(), new.Bytes.Hex()) + return found, new } // NthNextHmyExt return the Nth next pubkey of Harmony + allowlist nodes, next can be negative number diff --git a/consensus/validator.go b/consensus/validator.go index 2ccc453645..2b18003212 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -18,6 +18,7 @@ import ( ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { + recvMsg, err := consensus.parseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). @@ -26,6 +27,9 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Msg("[OnAnnounce] Unparseable leader message") return } + if consensus.ShardID == 0 { + //fmt.Println("onAnnounce called ", recvMsg.BlockNum) + } // NOTE let it handle its own logs if !consensus.onAnnounceSanityChecks(recvMsg) { diff --git a/consensus/view_change.go b/consensus/view_change.go index d7080c5a79..fb1b995f48 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "time" @@ -141,6 +142,7 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { Uint64("stuckBlockViewID", stuckBlockViewID). Msg("[getNextViewID]") + fmt.Println("end getNextViewID: ", nextViewID, viewChangeDuration) // duration is always the fixed view change duration for synchronous view change return nextViewID, viewChangeDuration } @@ -213,6 +215,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe lastLeaderPubKey, gap) } + fmt.Println("wasfoundNext", consensus.Blockchain.Config().IsAllowlistEpoch(epoch), wasFound, next.Bytes.Hex(), lastLeaderPubKey.Bytes.Hex()) if !wasFound { consensus.getLogger().Warn(). Str("key", consensus.LeaderPubKey.Bytes.Hex()). @@ -234,6 +237,7 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { + fmt.Printf("Message to send leader111: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } @@ -242,6 +246,7 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() + //fmt.Println("startViewChange", nextViewID) consensus.SetViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet @@ -282,7 +287,9 @@ func (consensus *Consensus) startViewChange() { if !consensus.isValidatorInCommittee(key.Pub.Bytes) { continue } + // Тут уже другой leader msgToSend := consensus.constructViewChangeMessage(&key) + fmt.Println("Message to send leader222: ", consensus.LeaderPubKey.Bytes.Hex()) if err := consensus.msgSender.SendWithRetry( consensus.BlockNum(), msg_pb.MessageType_VIEWCHANGE, @@ -340,6 +347,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.resetState() } + fmt.Println("[startNewView]", newLeaderPriKey.Pub.Bytes.Hex()) consensus.setLeaderPubKey(newLeaderPriKey.Pub) return nil diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index b940fbfeb4..592e8dc6b3 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -8,6 +8,7 @@ import ( "path" "runtime/debug" "strconv" + "strconv" "sync" "time" @@ -213,35 +214,3 @@ func GetPort() int { } return 0 } - -func AssertNoLongerThan0[E any](t time.Duration, f func() E) E { - ch := make(chan E) - defer close(ch) - stack := debug.Stack() - go func() { - select { - case <-time.After(t): - panic("AssertNoLongerThan0: " + string(stack)) - case <-ch: - return - } - }() - - return f() -} - -func AssertNoLongerThan(t time.Duration, f func()) { - ch := make(chan struct{}) - defer close(ch) - stack := debug.Stack() - go func() { - select { - case <-time.After(t): - panic("AssertNoLongerThan0: " + string(stack)) - case <-ch: - return - } - }() - - f() -} diff --git a/node/node_newblock.go b/node/node_newblock.go index c29fb1b96a..69523861f2 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -89,8 +89,6 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp newBlock, err := node.ProposeNewBlock(newCommitSigsChan) if err == nil { - fmt.Printf("ProposeNewBlock: #%d :%d e:%d, sh:%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), node.Blockchain().CurrentHeader().Epoch().Uint64(), node.Blockchain().ShardID(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) - if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). Msg("Block with the same number was already proposed, abort.") From 881e3b7ab8f2781fa638311db496dee16ffe78c7 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 5 Sep 2022 19:39:49 +0700 Subject: [PATCH 105/420] consensus check is forked --- api/service/explorer/service.go | 6 +++--- consensus/consensus.go | 2 -- internal/params/config.go | 2 -- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index d96c1bc56c..5887bb3a02 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -223,9 +223,9 @@ func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { for i := cur; i > 0; i-- { block := s.blockchain.GetBlockByNumber(i) - w.Write([]byte(fmt.Sprintf("%d ", i))) - w.Write([]byte(fmt.Sprintf("%s ", block.Header().ViewID().String()))) - w.Write([]byte(fmt.Sprintf("%s ", block.Header().Coinbase().Hash().Hex()))) + w.Write([]byte(fmt.Sprintf("#%d ", i))) + w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) + w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) } } diff --git a/consensus/consensus.go b/consensus/consensus.go index b73ecfb8e9..10144c9108 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -71,8 +71,6 @@ type Consensus struct { priKey multibls.PrivateKeys // the publickey of leader LeaderPubKey *bls.PublicKeyWrapper - // index of leader in the list of validators. - LeaderIndex int // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 diff --git a/internal/params/config.go b/internal/params/config.go index 725704bd06..f5ce665b1f 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -274,8 +274,6 @@ var ( LeaderRotationEpoch: big.NewInt(4), LeaderRotationBlocksCount: 5, FeeCollectEpoch: big.NewInt(5), - LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 5, } // AllProtocolChanges ... From f43add44c5aee293198c1ab4a3a361e5891ab8b4 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 5 Sep 2022 22:41:06 +0700 Subject: [PATCH 106/420] update master --- api/service/explorer/service.go | 2 ++ consensus/view_change.go | 2 +- node/node_newblock.go | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 5887bb3a02..1ebfa94ba1 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -223,6 +223,8 @@ func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { for i := cur; i > 0; i-- { block := s.blockchain.GetBlockByNumber(i) + leaderPubKey, _ := chain.GetLeaderPubKeyFromCoinbase(s.backend.Blockchain(), block.Header()) + w.Write([]byte(fmt.Sprintf("%s ", leaderPubKey.Bytes.Hex()))) w.Write([]byte(fmt.Sprintf("#%d ", i))) w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) diff --git a/consensus/view_change.go b/consensus/view_change.go index fb1b995f48..226db683c4 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -237,7 +237,7 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - fmt.Printf("Message to send leader111: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) + fmt.Printf("[startViewChange]: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } diff --git a/node/node_newblock.go b/node/node_newblock.go index 69523861f2..f96477413f 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -146,7 +146,6 @@ func (node *Node) ProposeNewBlock(commitSigs chan []byte) (*types.Block, error) if node.Blockchain().Config().IsStaking(header.Epoch()) { blsPubKeyBytes := leaderKey.Object.GetAddress() coinbase.SetBytes(blsPubKeyBytes[:]) - fmt.Println("coinbase.SetBytes leader: ", leaderKey.Bytes.Hex(), coinbase.Hex()) } emptyAddr := common.Address{} From 1b411964318c97581fc340e016e70d22b5e621dc Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 6 Sep 2022 02:58:48 +0700 Subject: [PATCH 107/420] fix leader --- consensus/consensus.go | 1 + consensus/consensus_service.go | 34 ++++++++++++++++++++++++++-------- consensus/consensus_v2.go | 2 +- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 10144c9108..33f6ee5a14 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -74,6 +74,7 @@ type Consensus struct { // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 + epoch uint64 // Blockhash - 32 byte blockHash [32]byte // Block to run consensus on diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 2fabb42b3c..76e5d9c44b 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -82,6 +82,7 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { consensus.Decider.UpdateParticipants(pubKeys, allowlist) + consensus.pubKeyLock.Unlock() consensus.getLogger().Info().Msg("My Committee updated") for i := range pubKeys { consensus.getLogger().Info(). @@ -89,15 +90,22 @@ func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.Publi Str("BLSPubKey", pubKeys[i].Bytes.Hex()). Msg("Member") } - - allKeys := consensus.Decider.Participants() - if len(allKeys) != 0 { - consensus.LeaderPubKey = &allKeys[0] - consensus.getLogger().Info(). - Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") + if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { + consensus.updateLeader() } else { - consensus.getLogger().Error(). - Msg("[UpdatePublicKeys] Participants is empty") + consensus.pubKeyLock.Lock() + allKeys := consensus.Decider.Participants() + consensus.pubKeyLock.Unlock() + if len(allKeys) != 0 { + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = &allKeys[0] + consensus.pubKeyLock.Unlock() + consensus.getLogger().Info(). + Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") + } else { + consensus.getLogger().Error(). + Msg("[UpdatePublicKeys] Participants is empty") + } } // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. @@ -658,3 +666,13 @@ func (consensus *Consensus) getLogger() *zerolog.Logger { Logger() return &logger } + +func UpdatePublicKeyDefault(consensus *Consensus) { + if allKeys := consensus.Decider.Participants(); len(allKeys) > 0 { + consensus.LeaderPubKey = &allKeys[0] + } +} + +func UpdatePublicKeyRotate(consensus *Consensus) { + //consensus +} diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index fea7b7dfe6..653ae72d66 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -778,7 +778,7 @@ func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, er if epoch.Uint64() == 0 { return 0, nil } - epochBlock := consensus.Blockchain.GetBlockByNumber(epoch.Uint64() - 1) + epochBlock := consensus.Blockchain.GetBlockByNumber(shard.Schedule.EpochLastBlock(epoch.Uint64() - 1)) if epochBlock == nil { return 0, errors.Errorf("block not found for number %d", epoch.Uint64()-1) } From a603b3ed975bdff444bd10de43e1d808e472638e Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 7 Sep 2022 22:20:05 +0700 Subject: [PATCH 108/420] check leader for N blocks --- consensus/consensus.go | 5 +++++ consensus/consensus_service.go | 31 +++++++++++++------------------ consensus/consensus_v2.go | 4 +--- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 33f6ee5a14..d7b86f1421 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -185,6 +185,11 @@ func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper { func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } +func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = pub + consensus.pubKeyLock.Unlock() +} func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 76e5d9c44b..5e553f2628 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -82,31 +82,26 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { consensus.Decider.UpdateParticipants(pubKeys, allowlist) + allKeys := consensus.Decider.Participants() consensus.pubKeyLock.Unlock() - consensus.getLogger().Info().Msg("My Committee updated") + if len(allKeys) != 0 { + first := consensus.Decider.FirstParticipant( + shard.Schedule.InstanceForEpoch(consensus.Blockchain.CurrentHeader().Epoch())) + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = first + consensus.pubKeyLock.Unlock() + consensus.getLogger().Info(). + Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") + } else { + consensus.getLogger().Error(). + Msg("[UpdatePublicKeys] Participants is empty") + } for i := range pubKeys { consensus.getLogger().Info(). Int("index", i). Str("BLSPubKey", pubKeys[i].Bytes.Hex()). Msg("Member") } - if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { - consensus.updateLeader() - } else { - consensus.pubKeyLock.Lock() - allKeys := consensus.Decider.Participants() - consensus.pubKeyLock.Unlock() - if len(allKeys) != 0 { - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = &allKeys[0] - consensus.pubKeyLock.Unlock() - consensus.getLogger().Info(). - Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") - } else { - consensus.getLogger().Error(). - Msg("[UpdatePublicKeys] Participants is empty") - } - } // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. consensus.updateBitmaps() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 653ae72d66..8a3d4110b2 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/hex" - "fmt" "math/big" "sync/atomic" "time" @@ -12,6 +11,7 @@ import ( "github.com/ethereum/go-ethereum/common" bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" + "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" @@ -437,7 +437,6 @@ func (consensus *Consensus) BlockChannel(newBlock *types.Block) { } // Sleep to wait for the full block time consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") - time.AfterFunc(time.Until(consensus.NextBlockDue), func() { consensus.StartFinalityCount() consensus.mutex.Lock() @@ -548,7 +547,6 @@ func (consensus *Consensus) getLastMileBlocksAndMsg(bnStart uint64) ([]*types.Bl // preCommitAndPropose commit the current block with 67% commit signatures and start // proposing new block which will wait on the full commit signatures to finish func (consensus *Consensus) preCommitAndPropose(blk *types.Block) error { - //fmt.Println("preCommitAndPropose", utils.GetPort(), blk.NumberU64()) if blk == nil { return errors.New("block to pre-commit is nil") } From 130c77426c5b8cc00d93fbd2e42f97873822c229 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sat, 10 Sep 2022 18:02:45 +0700 Subject: [PATCH 109/420] fix --- api/service/explorer/service.go | 2 -- cmd/harmony/main.go | 3 --- consensus/consensus_service.go | 11 ----------- consensus/quorum/quorum.go | 7 +------ 4 files changed, 1 insertion(+), 22 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 1ebfa94ba1..b13f73db61 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -115,8 +115,6 @@ func (s *Service) Run() *http.Server { s.router = mux.NewRouter() - fmt.Println("++", addr) - // Set up router for addresses. // Fetch addresses request, accepts parameter size: how much addresses to read, // parameter prefix: from which address prefix start diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 2fcfe5f037..4bb70a81b4 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -266,8 +266,6 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { var err error - fmt.Println("OS: ", os.Args) - nodeconfigSetShardSchedule(hc) nodeconfig.SetShardingSchedule(shard.Schedule) nodeconfig.SetVersion(getHarmonyVersion()) @@ -803,7 +801,6 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // Set the consensus ID to be the current block number viewID := currentNode.Blockchain().CurrentBlock().Header().ViewID().Uint64() - fmt.Println("viewID:", viewID) currentConsensus.SetViewIDs(viewID + 1) utils.Logger().Info(). Uint64("viewID", viewID). diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 5e553f2628..3f75c0bb00 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -232,7 +232,6 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } - fmt.Println("[checkViewID] Set LEADEER PUB KEY ", msg.SenderPubkeys[0].Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() @@ -661,13 +660,3 @@ func (consensus *Consensus) getLogger() *zerolog.Logger { Logger() return &logger } - -func UpdatePublicKeyDefault(consensus *Consensus) { - if allKeys := consensus.Decider.Participants(); len(allKeys) > 0 { - consensus.LeaderPubKey = &allKeys[0] - } -} - -func UpdatePublicKeyRotate(consensus *Consensus) { - //consensus -} diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 0586ace4fe..6cf79e549a 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -230,17 +230,12 @@ func (s *cIdentities) NthNextHmy(instance shardingconfig.Instance, pubKey *bls.P Msg("[NthNextHmy] pubKey not found") } numNodes := instance.NumHarmonyOperatedNodesPerShard() - //fmt.Println("??idx:", idx, numNodes) // sanity check to avoid out of bound access if numNodes <= 0 || numNodes > len(s.publicKeys) { numNodes = len(s.publicKeys) } idx = (idx + next) % numNodes - //fmt.Println("-------idx:", idx) - - new := &s.publicKeys[idx] - fmt.Println("NthNextHmy: ", pubKey.Bytes.Hex(), new.Bytes.Hex()) - return found, new + return found, &s.publicKeys[idx] } // NthNextHmyExt return the Nth next pubkey of Harmony + allowlist nodes, next can be negative number From aa908acd9c5be78e88f4681e3a208adccbaf449a Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 14:04:09 +0700 Subject: [PATCH 110/420] fix --- consensus/consensus_service.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 3f75c0bb00..d436ef5d56 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync/atomic" "time" @@ -496,7 +495,6 @@ func (consensus *Consensus) isLeader() bool { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { - fmt.Println("SetViewIDs", height) consensus.setViewIDs(height) } From 5927026d1b8717b049b1a0af70694fda14a3f466 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 15:27:56 +0700 Subject: [PATCH 111/420] Cleanup and fix update pub keys. --- consensus/consensus_service.go | 16 ++++++++++------ consensus/consensus_v2.go | 1 - consensus/leader.go | 1 - consensus/view_change.go | 7 ------- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index d436ef5d56..1913460eb9 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync/atomic" "time" @@ -81,14 +82,17 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { consensus.Decider.UpdateParticipants(pubKeys, allowlist) + consensus.getLogger().Info().Msg("My Committee updated") + for i := range pubKeys { + consensus.getLogger().Info(). + Int("index", i). + Str("BLSPubKey", pubKeys[i].Bytes.Hex()). + Msg("Member") + } + allKeys := consensus.Decider.Participants() - consensus.pubKeyLock.Unlock() if len(allKeys) != 0 { - first := consensus.Decider.FirstParticipant( - shard.Schedule.InstanceForEpoch(consensus.Blockchain.CurrentHeader().Epoch())) - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = first - consensus.pubKeyLock.Unlock() + consensus.LeaderPubKey = &allKeys[0] consensus.getLogger().Info(). Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") } else { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 8a3d4110b2..3ec460ef52 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -664,7 +664,6 @@ func (consensus *Consensus) tryCatchup() error { consensus.getLogger().Error().Err(err).Msg("[TryCatchup] Failed to add block to chain") return err } - //fmt.Println("tryCatchup ", utils.GetPort(), blk.NumberU64()) select { // TODO: Remove this when removing dns sync and stream sync is fully up case consensus.VerifiedNewBlock <- blk: diff --git a/consensus/leader.go b/consensus/leader.go index b34757ae64..548ec98c0a 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -322,5 +322,4 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED) } - //fmt.Println("onCommit99: ", utils.GetPort(), recvMsg.BlockNum) } diff --git a/consensus/view_change.go b/consensus/view_change.go index 226db683c4..377a7098f7 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "time" @@ -142,7 +141,6 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { Uint64("stuckBlockViewID", stuckBlockViewID). Msg("[getNextViewID]") - fmt.Println("end getNextViewID: ", nextViewID, viewChangeDuration) // duration is always the fixed view change duration for synchronous view change return nextViewID, viewChangeDuration } @@ -215,7 +213,6 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe lastLeaderPubKey, gap) } - fmt.Println("wasfoundNext", consensus.Blockchain.Config().IsAllowlistEpoch(epoch), wasFound, next.Bytes.Hex(), lastLeaderPubKey.Bytes.Hex()) if !wasFound { consensus.getLogger().Warn(). Str("key", consensus.LeaderPubKey.Bytes.Hex()). @@ -237,7 +234,6 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - fmt.Printf("[startViewChange]: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } @@ -246,7 +242,6 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() - //fmt.Println("startViewChange", nextViewID) consensus.SetViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet @@ -287,9 +282,7 @@ func (consensus *Consensus) startViewChange() { if !consensus.isValidatorInCommittee(key.Pub.Bytes) { continue } - // Тут уже другой leader msgToSend := consensus.constructViewChangeMessage(&key) - fmt.Println("Message to send leader222: ", consensus.LeaderPubKey.Bytes.Hex()) if err := consensus.msgSender.SendWithRetry( consensus.BlockNum(), msg_pb.MessageType_VIEWCHANGE, From 4595587494a1f74d66766b6653716b31ede666bf Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 16:32:36 +0700 Subject: [PATCH 112/420] Rotate leader. --- consensus/consensus.go | 1 - 1 file changed, 1 deletion(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index d7b86f1421..4cf4fdf6f8 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -74,7 +74,6 @@ type Consensus struct { // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 - epoch uint64 // Blockhash - 32 byte blockHash [32]byte // Block to run consensus on From 3a45cd2277c97f61d6b0703cdcd902d0b935fd4a Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 13 Sep 2022 01:10:13 +0700 Subject: [PATCH 113/420] fix fix fix fix fix --- consensus/consensus_service.go | 1 - consensus/consensus_v2.go | 6 ++++++ internal/params/config.go | 11 +++++++---- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 1913460eb9..411b9f6552 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync/atomic" "time" diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 3ec460ef52..e8fa0fb17c 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -759,6 +759,12 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg } else { epoch = blk.Epoch() } + var epoch *big.Int + if blk.IsLastBlockInEpoch() { + epoch = new(big.Int).Add(blk.Epoch(), common.Big1) + } else { + epoch = blk.Epoch() + } if consensus.Blockchain.Config().IsLeaderRotation(epoch) { consensus.rotateLeader(epoch) } diff --git a/internal/params/config.go b/internal/params/config.go index f5ce665b1f..310172dc71 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -274,6 +274,8 @@ var ( LeaderRotationEpoch: big.NewInt(4), LeaderRotationBlocksCount: 5, FeeCollectEpoch: big.NewInt(5), + LeaderRotationEpoch: big.NewInt(4), + LeaderRotationBlocksCount: 5, } // AllProtocolChanges ... @@ -313,8 +315,8 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } @@ -355,8 +357,9 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount + + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } From 6081a19a9adcd570a92d8aa6d04d4162aa6dbde9 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 13 Sep 2022 02:15:25 +0700 Subject: [PATCH 114/420] Cleaned. --- api/service/explorer/service.go | 48 ------------------------------- cmd/harmony/main.go | 3 +- consensus/consensus_v2.go | 13 --------- consensus/validator.go | 11 ------- consensus/view_change.go | 7 ----- node/node_newblock.go | 1 - test/configs/local-resharding.txt | 31 ++++++++++++++------ test/deploy.sh | 5 +--- 8 files changed, 24 insertions(+), 95 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index b13f73db61..b5ffa1b8b0 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -7,13 +7,11 @@ import ( "fmt" "net" "net/http" - "os" "path" "strconv" "time" "github.com/RoaringBitmap/roaring/roaring64" - "github.com/harmony-one/harmony/internal/common" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" ethCommon "github.com/ethereum/go-ethereum/common" @@ -121,9 +119,6 @@ func (s *Service) Run() *http.Server { s.router.Path("/addresses").Queries("size", "{[0-9]*?}", "prefix", "{[a-zA-Z0-9]*?}").HandlerFunc(s.GetAddresses).Methods("GET") s.router.Path("/addresses").HandlerFunc(s.GetAddresses) s.router.Path("/height").HandlerFunc(s.GetHeight) - s.router.Path("/leader").HandlerFunc(s.GetLeader) - s.router.Path("/blocks").HandlerFunc(s.GetBlocks) - s.router.Path("/halt").HandlerFunc(s.halt) // Set up router for supply info s.router.Path("/burn-addresses").Queries().HandlerFunc(s.GetInaccessibleAddressInfo).Methods("GET") @@ -191,49 +186,6 @@ type HeightResponse struct { S3 uint64 `json:"3,omitempty"` } -func (s *Service) GetLeader(w http.ResponseWriter, r *http.Request) { - if s.backend.IsCurrentlyLeader() { - w.Write([]byte("true ")) - } else { - w.Write([]byte("false")) - } - - keys := "" - for _, p := range s.backend.GetPublicKeys() { - addr := common.Address{} - addrBytes := p.Object.GetAddress() - addr.SetBytes(addrBytes[:]) - keys += fmt.Sprintf("%s ", addr.String()) - break - } - //blsPubKeyBytes := leaderKey.Object.GetAddress() - //coinbase.SetBytes(blsPubKeyBytes[:]) - - w.Write([]byte(fmt.Sprintf(" %d", s.blockchain.ShardID()))) - w.Write([]byte(fmt.Sprintf(" %s", s.Port))) - w.Write([]byte(fmt.Sprintf(" %s", keys))) - w.Write([]byte(fmt.Sprintf(" %s", s.backend.GetPublicKeys().SerializeToHexStr()))) - -} - -func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { - cur := s.blockchain.CurrentHeader().Number().Uint64() - - for i := cur; i > 0; i-- { - block := s.blockchain.GetBlockByNumber(i) - leaderPubKey, _ := chain.GetLeaderPubKeyFromCoinbase(s.backend.Blockchain(), block.Header()) - w.Write([]byte(fmt.Sprintf("%s ", leaderPubKey.Bytes.Hex()))) - w.Write([]byte(fmt.Sprintf("#%d ", i))) - w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) - w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) - w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) - } -} - -func (s *Service) halt(w http.ResponseWriter, r *http.Request) { - os.Exit(0) -} - // GetHeight returns heights of current and beacon chains if needed. func (s *Service) GetHeight(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 4bb70a81b4..eacc392f10 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -425,9 +425,8 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { if currentNode.NodeConfig.Role() == nodeconfig.Validator { currentNode.RegisterValidatorServices() } else if currentNode.NodeConfig.Role() == nodeconfig.ExplorerNode { - + currentNode.RegisterExplorerServices() } - currentNode.RegisterExplorerServices() currentNode.RegisterService(service.CrosslinkSending, crosslink_sending.New(currentNode, currentNode.Blockchain())) if hc.Pprof.Enabled { setupPprofService(currentNode, hc) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index e8fa0fb17c..19d0c88907 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -678,8 +678,6 @@ func (consensus *Consensus) tryCatchup() error { } func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMessage) error { - // this function evaluates for all, leader and validators. - if consensus.Blockchain().CurrentBlock().NumberU64() < blk.NumberU64() { if _, err := consensus.Blockchain().InsertChain([]*types.Block{blk}, !consensus.FBFTLog.IsBlockVerified(blk.Hash())); err != nil { consensus.getLogger().Error().Err(err).Msg("[commitBlock] Failed to add block to chain") @@ -777,17 +775,6 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg consensus.resetState() } -func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, error) { - if epoch.Uint64() == 0 { - return 0, nil - } - epochBlock := consensus.Blockchain.GetBlockByNumber(shard.Schedule.EpochLastBlock(epoch.Uint64() - 1)) - if epochBlock == nil { - return 0, errors.Errorf("block not found for number %d", epoch.Uint64()-1) - } - return epochBlock.Header().ViewID().Uint64() + 1, nil -} - func (consensus *Consensus) postCatchup(initBN uint64) { if initBN < consensus.BlockNum() { consensus.getLogger().Info(). diff --git a/consensus/validator.go b/consensus/validator.go index 2b18003212..06aa9bc08c 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -18,7 +18,6 @@ import ( ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { - recvMsg, err := consensus.parseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). @@ -27,9 +26,6 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Msg("[OnAnnounce] Unparseable leader message") return } - if consensus.ShardID == 0 { - //fmt.Println("onAnnounce called ", recvMsg.BlockNum) - } // NOTE let it handle its own logs if !consensus.onAnnounceSanityChecks(recvMsg) { @@ -398,13 +394,6 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { consensus.getLogger().Info().Msg("[OnCommitted] Start consensus timer (new block added)") consensus.consensusTimeout[timeoutConsensus].Start() } - - //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum) - if blk != nil { - //consensus.ReshardingNextLeader(blk) - } else { - //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum, "blk is nil") - } } // Collect private keys that are part of the current committee. diff --git a/consensus/view_change.go b/consensus/view_change.go index 377a7098f7..06640c123d 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -367,13 +367,6 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { return } - consensus.getLogger().Debug(). - Err(err). - Interface("SenderPubkeys", recvMsg.SenderPubkeys). - Str("NextLeader", recvMsg.LeaderPubkey.Bytes.Hex()). - Str("myBLSPubKey", consensus.priKey.GetPublicKeys().SerializeToHexStr()). - Msg("[onViewChange] I am the Leader") - if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) { consensus.getLogger().Info(). Int64("have", consensus.Decider.SignersCount(quorum.ViewChange)). diff --git a/node/node_newblock.go b/node/node_newblock.go index f96477413f..5050e4d6a7 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -2,7 +2,6 @@ package node import ( "errors" - "fmt" "sort" "strings" "time" diff --git a/test/configs/local-resharding.txt b/test/configs/local-resharding.txt index fc3d3e97b4..4ce91ece1c 100644 --- a/test/configs/local-resharding.txt +++ b/test/configs/local-resharding.txt @@ -1,12 +1,25 @@ 127.0.0.1 9000 validator .hmy/65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204.key -127.0.0.1 9002 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key -127.0.0.1 9004 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key -127.0.0.1 9006 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key -127.0.0.1 9008 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key -127.0.0.1 9010 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key -127.0.0.1 9099 explorer null 0 +127.0.0.1 9002 validator .hmy/40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809.key +127.0.0.1 9004 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key +127.0.0.1 9006 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key +127.0.0.1 9008 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key +127.0.0.1 9010 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key +127.0.0.1 9012 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key +127.0.0.1 9014 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key +127.0.0.1 9016 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key +127.0.0.1 9018 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key +127.0.0.1 9020 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key +127.0.0.1 9022 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key +127.0.0.1 9200 explorer null 0 127.0.0.1 9100 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key -127.0.0.1 9102 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key -127.0.0.1 9104 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key -127.0.0.1 9106 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key \ No newline at end of file +127.0.0.1 9102 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key +127.0.0.1 9104 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key +127.0.0.1 9106 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key +127.0.0.1 9108 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key +127.0.0.1 9110 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key +127.0.0.1 9112 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key +127.0.0.1 9114 validator .hmy/f47238daef97d60deedbde5302d05dea5de67608f11f406576e363661f7dcbc4a1385948549b31a6c70f6fde8a391486.key +127.0.0.1 9116 validator .hmy/fc4b9c535ee91f015efff3f32fbb9d32cdd9bfc8a837bb3eee89b8fff653c7af2050a4e147ebe5c7233dc2d5df06ee0a.key +127.0.0.1 9118 validator .hmy/ca86e551ee42adaaa6477322d7db869d3e203c00d7b86c82ebee629ad79cb6d57b8f3db28336778ec2180e56a8e07296.key +127.0.0.1 9300 explorer null 1 \ No newline at end of file diff --git a/test/deploy.sh b/test/deploy.sh index fe22de57f0..9ab6943110 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -69,7 +69,7 @@ function launch_localnet() { if ${VERBOSE}; then verbosity=5 else - verbosity=5 + verbosity=3 fi base_args=(--log_folder "${log_folder}" --min_peers "${MIN}" --bootnodes "${BN_MA}" "--network_type=$NETWORK" --blspass file:"${ROOT}/.hmy/blspass.txt" "--dns=false" "--verbosity=${verbosity}" "--p2p.security.max-conn-per-ip=100") @@ -80,11 +80,8 @@ function launch_localnet() { while IFS='' read -r line || [[ -n "$line" ]]; do i=$((i + 1)) - - # Read config for i-th node form config file IFS=' ' read -r ip port mode bls_key shard node_config <<<"${line}" - echo "LINE: ${line} ${shard}" args=("${base_args[@]}" --ip "${ip}" --port "${port}" --key "/tmp/${ip}-${port}.key" --db_dir "${ROOT}/db-${ip}-${port}" "--broadcast_invalid_tx=false") if [[ -z "$ip" || -z "$port" ]]; then echo "skip empty node" From 8360cbd80c6719dd2cc33ddc7cd9d90608f0c1e0 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 19 Sep 2022 21:40:21 +0700 Subject: [PATCH 115/420] Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. --- consensus/consensus_v2.go | 1 - 1 file changed, 1 deletion(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 19d0c88907..1fcb6e2e4b 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -11,7 +11,6 @@ import ( "github.com/ethereum/go-ethereum/common" bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" - "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" From 3ded0f8ff882f88c7ab4e4550f22a850ccd4cc62 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 17 Oct 2022 17:09:04 +0700 Subject: [PATCH 116/420] comment activation --- internal/params/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/params/config.go b/internal/params/config.go index 310172dc71..74f32483b3 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -110,7 +110,7 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(2), AllowlistEpoch: big.NewInt(2), LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 64, + LeaderRotationBlocksCount: 64, FeeCollectEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. From 5aea88bf087eaede787f7f72e6812c7046299924 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 18 Oct 2022 12:17:17 +0700 Subject: [PATCH 117/420] 295 epoch --- internal/params/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/params/config.go b/internal/params/config.go index 74f32483b3..310172dc71 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -110,7 +110,7 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(2), AllowlistEpoch: big.NewInt(2), LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 64, + LeaderRotationBlocksCount: 64, FeeCollectEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. From 965ceb92c3a2a1a38969a03f38f37eb9eb43b951 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 8 Nov 2022 20:19:37 +0700 Subject: [PATCH 118/420] Fix failed tests. --- internal/params/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/params/config.go b/internal/params/config.go index 310172dc71..6168dc9bbd 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -315,8 +315,8 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } @@ -360,7 +360,7 @@ var ( big.NewInt(1), // LeaderRotationEpoch 64, // LeaderRotationBlocksCount - big.NewInt(0), // FeeCollectEpoch + big.NewInt(0), // FeeCollectEpoch } // TestRules ... From 73953f0830713658de72f397f1f98eab710a5ab6 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 9 Nov 2022 15:36:28 +0700 Subject: [PATCH 119/420] Fixed code review. --- consensus/consensus_v2.go | 3 +-- hmy/hmy.go | 2 -- internal/params/config.go | 9 ++++----- node/api.go | 6 ------ 4 files changed, 5 insertions(+), 15 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 1fcb6e2e4b..e6e6445914 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -137,7 +137,6 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb } func (consensus *Consensus) finalCommit() { - // THIS IS NOT GOOD PLACE FOR LEADER SWITCHING numCommits := consensus.Decider.SignersCount(quorum.Commit) consensus.getLogger().Info(). @@ -714,7 +713,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { if header == nil { return } - // Previous epoch, we should not change leader. + // Previous block was epoch block, we should not change leader. if header.Epoch().Uint64() != epoch.Uint64() { return } diff --git a/hmy/hmy.go b/hmy/hmy.go index 8089f04ba8..58e8d59edf 100644 --- a/hmy/hmy.go +++ b/hmy/hmy.go @@ -18,7 +18,6 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" - "github.com/harmony-one/harmony/multibls" commonRPC "github.com/harmony-one/harmony/rpc/common" "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" @@ -97,7 +96,6 @@ type NodeAPI interface { GetStakingTransactionsCount(address, txType string) (uint64, error) GetTraceResultByHash(hash common.Hash) (json.RawMessage, error) IsCurrentlyLeader() bool - GetPublicKeys() multibls.PublicKeys IsOutOfSync(shardID uint32) bool SyncStatus(shardID uint32) (bool, uint64, uint64) SyncPeers() map[string]int diff --git a/internal/params/config.go b/internal/params/config.go index 6168dc9bbd..725704bd06 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -274,7 +274,7 @@ var ( LeaderRotationEpoch: big.NewInt(4), LeaderRotationBlocksCount: 5, FeeCollectEpoch: big.NewInt(5), - LeaderRotationEpoch: big.NewInt(4), + LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 5, } @@ -357,10 +357,9 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount - big.NewInt(0), // FeeCollectEpoch + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount + big.NewInt(0), // FeeCollectEpoch } // TestRules ... diff --git a/node/api.go b/node/api.go index debcb201f6..ceda968084 100644 --- a/node/api.go +++ b/node/api.go @@ -6,7 +6,6 @@ import ( "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy" "github.com/harmony-one/harmony/internal/tikv" - "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/rosetta" hmy_rpc "github.com/harmony-one/harmony/rpc" rpc_common "github.com/harmony-one/harmony/rpc/common" @@ -19,11 +18,6 @@ func (node *Node) IsCurrentlyLeader() bool { return node.Consensus.IsLeader() } -// GetPublicKeys exposes if node is currently the leader node -func (node *Node) GetPublicKeys() multibls.PublicKeys { - return node.Consensus.GetPrivateKeys().GetPublicKeys() -} - // PeerConnectivity .. func (node *Node) PeerConnectivity() (int, int, int) { return node.host.PeerConnectivity() From bd8326971729d5f53fd577cd7fe0d16d88cf1411 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 10 Nov 2022 12:57:57 +0700 Subject: [PATCH 120/420] Fix review comments. --- consensus/consensus.go | 5 +++++ consensus/consensus_v2.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 4cf4fdf6f8..1c9430caa5 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -184,6 +184,11 @@ func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper { func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } + +func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { + return consensus.LeaderPubKey +} + func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { consensus.pubKeyLock.Lock() consensus.LeaderPubKey = pub diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index e6e6445914..e9cdfe5ef4 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -713,7 +713,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { if header == nil { return } - // Previous block was epoch block, we should not change leader. + // Previous epoch, we should not change leader. if header.Epoch().Uint64() != epoch.Uint64() { return } From 5504a7345553f02b6d78265b88682e0c219f209c Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 17 Jan 2023 13:22:30 -0300 Subject: [PATCH 121/420] Merged leader rotation. --- api/service/stagedsync/stagedsync.go | 26 ++++++++++++-------------- consensus/consensus_service.go | 12 ------------ consensus/consensus_v2.go | 15 +++++---------- internal/params/config.go | 2 -- internal/utils/singleton.go | 2 -- 5 files changed, 17 insertions(+), 40 deletions(-) diff --git a/api/service/stagedsync/stagedsync.go b/api/service/stagedsync/stagedsync.go index aca707f619..83af6abf9f 100644 --- a/api/service/stagedsync/stagedsync.go +++ b/api/service/stagedsync/stagedsync.go @@ -1205,22 +1205,20 @@ func (ss *StagedSync) GetMaxPeerHeight() (uint64, error) { return ss.getMaxPeerHeight() } -func (ss *StagedSync) addConsensusLastMile(bc core.BlockChain, consensus *consensus.Consensus) error { +func (ss *StagedSync) addConsensusLastMile(bc core.BlockChain, cs *consensus.Consensus) error { curNumber := bc.CurrentBlock().NumberU64() - blockIter, err := consensus.GetLastMileBlockIter(curNumber + 1) - if err != nil { - return err - } - for { - block := blockIter.Next() - if block == nil { - break - } - if _, err := bc.InsertChain(types.Blocks{block}, true); err != nil { - return errors.Wrap(err, "failed to InsertChain") + return cs.GetLastMileBlockIter(curNumber+1, func(blockIter *consensus.LastMileBlockIter) error { + for { + block := blockIter.Next() + if block == nil { + break + } + if _, err := bc.InsertChain(types.Blocks{block}, true); err != nil { + return errors.Wrap(err, "failed to InsertChain") + } } - } - return nil + return nil + }) } // GetSyncingPort returns the syncing port. diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 411b9f6552..e62b2ed048 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -471,18 +471,6 @@ func (consensus *Consensus) IsLeader() bool { return consensus.isLeader() } -// IsLeader check if the node is a leader or not by comparing the public key of -// the node with the leader public key -func (consensus *Consensus) isLeader() bool { - obj := consensus.LeaderPubKey.Object - for _, key := range consensus.priKey { - if key.Pub.Object.IsEqual(obj) { - return true - } - } - return false -} - // isLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key. This function assume it runs under lock. func (consensus *Consensus) isLeader() bool { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index e9cdfe5ef4..3ef0ea0655 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -412,16 +412,6 @@ func (consensus *Consensus) tick() { } } -// Close closes the consensus. If current is in normal commit phase, wait until the commit -// phase end. -func (consensus *Consensus) Close() error { - if consensus.dHelper != nil { - consensus.dHelper.close() - } - consensus.waitForCommit() - return nil -} - func (consensus *Consensus) BlockChannel(newBlock *types.Block) { //consensus.ReshardingNextLeader(newBlock) consensus.getLogger().Info(). @@ -489,6 +479,11 @@ func (consensus *Consensus) GetLastMileBlockIter(bnStart uint64, cb func(iter *L consensus.mutex.Lock() defer consensus.mutex.Unlock() + return consensus.getLastMileBlockIter(bnStart, cb) +} + +// GetLastMileBlockIter get the iterator of the last mile blocks starting from number bnStart +func (consensus *Consensus) getLastMileBlockIter(bnStart uint64, cb func(iter *LastMileBlockIter) error) error { if consensus.BlockVerifier == nil { return errors.New("consensus haven't initialized yet") } diff --git a/internal/params/config.go b/internal/params/config.go index 725704bd06..f5ce665b1f 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -274,8 +274,6 @@ var ( LeaderRotationEpoch: big.NewInt(4), LeaderRotationBlocksCount: 5, FeeCollectEpoch: big.NewInt(5), - LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 5, } // AllProtocolChanges ... diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index 592e8dc6b3..10101d7673 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -6,8 +6,6 @@ import ( "fmt" "os" "path" - "runtime/debug" - "strconv" "strconv" "sync" "time" From b1df6ec8cb70342b83153b81d0c299f971448aa7 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sat, 21 Jan 2023 17:07:40 -0300 Subject: [PATCH 122/420] Rebased on dev. --- api/service/consensus/service.go | 2 +- consensus/consensus.go | 8 ++------ consensus/consensus_service.go | 16 +--------------- consensus/consensus_v2.go | 20 +++++++------------- consensus/view_change.go | 14 ++++++++++---- 5 files changed, 21 insertions(+), 39 deletions(-) diff --git a/api/service/consensus/service.go b/api/service/consensus/service.go index d85a072f2e..49a1da64fb 100644 --- a/api/service/consensus/service.go +++ b/api/service/consensus/service.go @@ -32,5 +32,5 @@ func (s *Service) Stop() error { utils.Logger().Info().Msg("Stopping consensus service.") close(s.stopChan) utils.Logger().Info().Msg("Consensus service stopped.") - return s.consensus.Close() + return nil } diff --git a/consensus/consensus.go b/consensus/consensus.go index 1c9430caa5..bbf3634be9 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -178,6 +178,8 @@ func (consensus *Consensus) getPublicKeys() multibls.PublicKeys { } func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() return consensus.getLeaderPubKey() } @@ -189,12 +191,6 @@ func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } -func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = pub - consensus.pubKeyLock.Unlock() -} - func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e62b2ed048..a92a71f7e4 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -111,7 +111,7 @@ func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.Publi // do not reset view change state if it is in view changing mode if !consensus.isViewChangingMode() { - consensus.ResetViewChangeState() + consensus.resetViewChangeState() } return consensus.Decider.ParticipantsCount() } @@ -501,19 +501,6 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } -// SetLeaderIndex set the leader index. -func (consensus *Consensus) SetLeaderIndex(f func(int) int) (current int) { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() - consensus.LeaderIndex = f(consensus.LeaderIndex) - return consensus.LeaderIndex -} - -func (consensus *Consensus) GetLeaderIndex() int { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() - return consensus.LeaderIndex -} // SetCurBlockViewID set the current view ID func (consensus *Consensus) setCurBlockViewID(viewID uint64) { consensus.current.SetCurBlockViewID(viewID) @@ -524,7 +511,6 @@ func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) } - // SetViewChangingID set the current view change ID func (consensus *Consensus) setViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 3ef0ea0655..23c51ba248 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -414,18 +414,18 @@ func (consensus *Consensus) tick() { func (consensus *Consensus) BlockChannel(newBlock *types.Block) { //consensus.ReshardingNextLeader(newBlock) - consensus.getLogger().Info(). - Uint64("MsgBlockNum", newBlock.NumberU64()). - Msg("[ConsensusMainLoop] Received Proposed New Block!") + consensus.getLogger().Info(). + Uint64("MsgBlockNum", newBlock.NumberU64()). + Msg("[ConsensusMainLoop] Received Proposed New Block!") if newBlock.NumberU64() < consensus.BlockNum() { consensus.getLogger().Warn().Uint64("newBlockNum", newBlock.NumberU64()). Msg("[ConsensusMainLoop] received old block, abort") return - } - // Sleep to wait for the full block time - consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") - time.AfterFunc(time.Until(consensus.NextBlockDue), func() { + } + // Sleep to wait for the full block time + consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") + time.AfterFunc(time.Until(consensus.NextBlockDue), func() { consensus.StartFinalityCount() consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -750,12 +750,6 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg } else { epoch = blk.Epoch() } - var epoch *big.Int - if blk.IsLastBlockInEpoch() { - epoch = new(big.Int).Add(blk.Epoch(), common.Big1) - } else { - epoch = blk.Epoch() - } if consensus.Blockchain.Config().IsLeaderRotation(epoch) { consensus.rotateLeader(epoch) } diff --git a/consensus/view_change.go b/consensus/view_change.go index 06640c123d..3ab3d2c9da 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -327,8 +327,8 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri consensus.current.SetMode(Normal) consensus.consensusTimeout[timeoutViewChange].Stop() - consensus.SetViewIDs(viewID) - consensus.ResetViewChangeState() + consensus.setViewIDs(viewID) + consensus.resetViewChangeState() consensus.consensusTimeout[timeoutConsensus].Start() consensus.getLogger().Info(). @@ -340,7 +340,6 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.resetState() } - fmt.Println("[startNewView]", newLeaderPriKey.Pub.Bytes.Hex()) consensus.setLeaderPubKey(newLeaderPriKey.Pub) return nil @@ -532,7 +531,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // newView message verified success, override my state consensus.setViewIDs(recvMsg.ViewID) consensus.LeaderPubKey = senderKey - consensus.ResetViewChangeState() + consensus.resetViewChangeState() consensus.msgSender.StopRetry(msg_pb.MessageType_VIEWCHANGE) @@ -553,6 +552,13 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // ResetViewChangeState resets the view change structure func (consensus *Consensus) ResetViewChangeState() { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + consensus.resetViewChangeState() +} + +// ResetViewChangeState resets the view change structure +func (consensus *Consensus) resetViewChangeState() { consensus.getLogger().Info(). Str("Phase", consensus.phase.String()). Msg("[ResetViewChangeState] Resetting view change state") From 177a09060094148fd73abd7fbea182ccb90977df Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Fri, 27 Jan 2023 01:52:17 -0300 Subject: [PATCH 123/420] Rebased on dev. --- consensus/consensus.go | 12 ------------ consensus/consensus_v2.go | 2 +- internal/params/config.go | 4 ---- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index bbf3634be9..080b39b33e 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -187,18 +187,6 @@ func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } -func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { - return consensus.LeaderPubKey -} - -func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { - return consensus.LeaderPubKey -} - -func (consensus *Consensus) setLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { - consensus.LeaderPubKey = pub -} - func (consensus *Consensus) setLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { consensus.LeaderPubKey = pub } diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 23c51ba248..44c3ebbd21 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -750,7 +750,7 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg } else { epoch = blk.Epoch() } - if consensus.Blockchain.Config().IsLeaderRotation(epoch) { + if consensus.Blockchain().Config().IsLeaderRotation(epoch) { consensus.rotateLeader(epoch) } diff --git a/internal/params/config.go b/internal/params/config.go index f5ce665b1f..8ee199d22a 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -711,10 +711,6 @@ func (c *ChainConfig) IsLeaderRotation(epoch *big.Int) bool { return isForked(c.LeaderRotationEpoch, epoch) } -func (c *ChainConfig) IsLeaderRotation(epoch *big.Int) bool { - return isForked(c.LeaderRotationEpoch, epoch) -} - // IsFeeCollectEpoch determines whether Txn Fees will be collected into the community-managed account. func (c *ChainConfig) IsFeeCollectEpoch(epoch *big.Int) bool { return isForked(c.FeeCollectEpoch, epoch) From 1fa5953deb6930eaa23a53c3cafba24f3010bd3d Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Fri, 27 Jan 2023 19:14:09 -0300 Subject: [PATCH 124/420] Fix usage of private methods. --- consensus/checks.go | 5 +++-- consensus/consensus.go | 4 ++++ consensus/consensus_service.go | 13 +++++++++++-- consensus/consensus_v2.go | 5 +++-- consensus/debug.go | 14 ++++++++++++++ consensus/validator.go | 2 +- consensus/view_change.go | 12 ++++++------ consensus/view_change_msg.go | 10 +++++----- core/blockchain_impl.go | 15 +++++++++++++-- 9 files changed, 60 insertions(+), 20 deletions(-) diff --git a/consensus/checks.go b/consensus/checks.go index 9e095a45c4..e4c9a7084f 100644 --- a/consensus/checks.go +++ b/consensus/checks.go @@ -56,9 +56,10 @@ func (consensus *Consensus) senderKeySanityChecks(msg *msg_pb.Message, senderKey } func (consensus *Consensus) isRightBlockNumAndViewID(recvMsg *FBFTMessage) bool { - if recvMsg.ViewID != consensus.GetCurBlockViewID() || recvMsg.BlockNum != consensus.BlockNum() { + blockNum := consensus.getBlockNum() + if recvMsg.ViewID != consensus.getCurBlockViewID() || recvMsg.BlockNum != blockNum { consensus.getLogger().Debug(). - Uint64("blockNum", consensus.BlockNum()). + Uint64("blockNum", blockNum). Str("recvMsg", recvMsg.String()). Msg("BlockNum/viewID not match") return false diff --git a/consensus/consensus.go b/consensus/consensus.go index 080b39b33e..2dbfa52a0c 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -224,6 +224,10 @@ func (consensus *Consensus) BlockNum() uint64 { return atomic.LoadUint64(&consensus.blockNum) } +func (consensus *Consensus) getBlockNum() uint64 { + return atomic.LoadUint64(&consensus.blockNum) +} + // New create a new Consensus record func New( host p2p.Host, shard uint32, multiBLSPriKey multibls.PrivateKeys, diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index a92a71f7e4..c43c7fe127 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -486,6 +486,8 @@ func (consensus *Consensus) isLeader() bool { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() consensus.setViewIDs(height) } @@ -625,11 +627,18 @@ func (consensus *Consensus) NumSignaturesIncludedInBlock(block *types.Block) uin return count } +// GetLogger returns logger for consensus contexts added. +func (consensus *Consensus) GetLogger() *zerolog.Logger { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() + return consensus.getLogger() +} + // getLogger returns logger for consensus contexts added func (consensus *Consensus) getLogger() *zerolog.Logger { logger := utils.Logger().With(). - Uint64("myBlock", consensus.BlockNum()). - Uint64("myViewID", consensus.GetCurBlockViewID()). + Uint64("myBlock", consensus.blockNum). + Uint64("myViewID", consensus.getCurBlockViewID()). Str("phase", consensus.phase.String()). Str("mode", consensus.current.Mode().String()). Logger() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 44c3ebbd21..1ab33d0bbf 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -46,6 +46,8 @@ const ( // IsViewChangingMode return true if curernt mode is viewchanging func (consensus *Consensus) IsViewChangingMode() bool { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() return consensus.isViewChangingMode() } @@ -413,8 +415,7 @@ func (consensus *Consensus) tick() { } func (consensus *Consensus) BlockChannel(newBlock *types.Block) { - //consensus.ReshardingNextLeader(newBlock) - consensus.getLogger().Info(). + consensus.GetLogger().Info(). Uint64("MsgBlockNum", newBlock.NumberU64()). Msg("[ConsensusMainLoop] Received Proposed New Block!") diff --git a/consensus/debug.go b/consensus/debug.go index ae1d2d92bf..da3587710a 100644 --- a/consensus/debug.go +++ b/consensus/debug.go @@ -12,10 +12,24 @@ func (consensus *Consensus) GetConsensusMode() string { // GetCurBlockViewID returns the current view ID of the consensus func (consensus *Consensus) GetCurBlockViewID() uint64 { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() + return consensus.getCurBlockViewID() +} + +// GetCurBlockViewID returns the current view ID of the consensus +func (consensus *Consensus) getCurBlockViewID() uint64 { return consensus.current.GetCurBlockViewID() } // GetViewChangingID returns the current view changing ID of the consensus func (consensus *Consensus) GetViewChangingID() uint64 { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() + return consensus.current.GetViewChangingID() +} + +// GetViewChangingID returns the current view changing ID of the consensus +func (consensus *Consensus) getViewChangingID() uint64 { return consensus.current.GetViewChangingID() } diff --git a/consensus/validator.go b/consensus/validator.go index 06aa9bc08c..f85cb8e3d8 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -277,7 +277,7 @@ func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { if committedMsg != nil { consensus.onCommitted(committedMsg) } - if curBlockNum < consensus.BlockNum() { + if curBlockNum < consensus.getBlockNum() { consensus.getLogger().Info().Msg("[OnPrepared] Successfully caught up with committed message") break } diff --git a/consensus/view_change.go b/consensus/view_change.go index 3ab3d2c9da..13acb86ae2 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -253,7 +253,7 @@ func (consensus *Consensus) startViewChange() { consensus.getLogger().Warn(). Uint64("nextViewID", nextViewID). - Uint64("viewChangingID", consensus.GetViewChangingID()). + Uint64("viewChangingID", consensus.getViewChangingID()). Dur("timeoutDuration", duration). Str("NextLeader", consensus.LeaderPubKey.Bytes.Hex()). Msg("[startViewChange]") @@ -270,7 +270,7 @@ func (consensus *Consensus) startViewChange() { if err := consensus.vc.InitPayload( consensus.FBFTLog, nextViewID, - consensus.BlockNum(), + consensus.getBlockNum(), consensus.priKey, members); err != nil { consensus.getLogger().Error().Err(err).Msg("[startViewChange] Init Payload Error") @@ -284,7 +284,7 @@ func (consensus *Consensus) startViewChange() { } msgToSend := consensus.constructViewChangeMessage(&key) if err := consensus.msgSender.SendWithRetry( - consensus.BlockNum(), + consensus.getBlockNum(), msg_pb.MessageType_VIEWCHANGE, []nodeconfig.GroupID{ nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, @@ -310,7 +310,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri } if err := consensus.msgSender.SendWithRetry( - consensus.BlockNum(), + consensus.getBlockNum(), msg_pb.MessageType_NEWVIEW, []nodeconfig.GroupID{ nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, @@ -450,10 +450,10 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { Msg("[onNewView] Received NewView Message") // change view and leaderKey to keep in sync with network - if consensus.BlockNum() != recvMsg.BlockNum { + if consensus.getBlockNum() != recvMsg.BlockNum { consensus.getLogger().Warn(). Uint64("MsgBlockNum", recvMsg.BlockNum). - Uint64("myBlockNum", consensus.BlockNum()). + Uint64("myBlockNum", consensus.getBlockNum()). Msg("[onNewView] Invalid block number") return } diff --git a/consensus/view_change_msg.go b/consensus/view_change_msg.go index 2433abdcbe..c241450306 100644 --- a/consensus/view_change_msg.go +++ b/consensus/view_change_msg.go @@ -23,8 +23,8 @@ func (consensus *Consensus) constructViewChangeMessage(priKey *bls.PrivateKeyWra Type: msg_pb.MessageType_VIEWCHANGE, Request: &msg_pb.Message_Viewchange{ Viewchange: &msg_pb.ViewChangeRequest{ - ViewId: consensus.GetViewChangingID(), - BlockNum: consensus.BlockNum(), + ViewId: consensus.getViewChangingID(), + BlockNum: consensus.getBlockNum(), ShardId: consensus.ShardID, SenderPubkey: priKey.Pub.Bytes[:], LeaderPubkey: consensus.LeaderPubKey.Bytes[:], @@ -33,7 +33,7 @@ func (consensus *Consensus) constructViewChangeMessage(priKey *bls.PrivateKeyWra } preparedMsgs := consensus.FBFTLog.GetMessagesByTypeSeq( - msg_pb.MessageType_PREPARED, consensus.BlockNum(), + msg_pb.MessageType_PREPARED, consensus.getBlockNum(), ) preparedMsg := consensus.FBFTLog.FindMessageByMaxViewID(preparedMsgs) @@ -83,7 +83,7 @@ func (consensus *Consensus) constructViewChangeMessage(priKey *bls.PrivateKeyWra } viewIDBytes := make([]byte, 8) - binary.LittleEndian.PutUint64(viewIDBytes, consensus.GetViewChangingID()) + binary.LittleEndian.PutUint64(viewIDBytes, consensus.getViewChangingID()) sign1 := priKey.Pri.SignHash(viewIDBytes) if sign1 != nil { vcMsg.ViewidSig = sign1.Serialize() @@ -107,7 +107,7 @@ func (consensus *Consensus) constructNewViewMessage(viewID uint64, priKey *bls.P Request: &msg_pb.Message_Viewchange{ Viewchange: &msg_pb.ViewChangeRequest{ ViewId: viewID, - BlockNum: consensus.BlockNum(), + BlockNum: consensus.getBlockNum(), ShardId: consensus.ShardID, SenderPubkey: priKey.Pub.Bytes[:], }, diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 783a571e89..8b0683bd65 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -327,8 +327,19 @@ func VerifyBlockCrossLinks(blockchain BlockChain, block *types.Block) error { // ReadCrossLink beacon chain usage. cl, err := blockchain.ReadCrossLink(crossLink.ShardID(), crossLink.BlockNum()) if err == nil && cl != nil { - // Add slash for exist same blocknum but different crosslink - return errAlreadyExist + utils.Logger().Err(errAlreadyExist). + Uint64("beacon-block-number", block.NumberU64()). + Interface("remote", crossLink). + Interface("local", cl). + Msg("[CrossLinkVerification]") + // TODO Add slash for exist same blocknum but different crosslink + return errors.Wrapf( + errAlreadyExist, + "[CrossLinkVerification] shard: %d block: %d on beacon block %d", + crossLink.ShardID(), + crossLink.BlockNum(), + block.NumberU64(), + ) } if err := VerifyCrossLink(blockchain, crossLink); err != nil { return errors.Wrapf(err, "cannot VerifyBlockCrossLinks") From abf5f5aa8c056510b49ebfb9f6f089538294b372 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sat, 28 Jan 2023 00:21:05 -0300 Subject: [PATCH 125/420] Fix usage of private methods. --- consensus/checks.go | 2 +- consensus/consensus_service.go | 12 ++++++++++++ consensus/consensus_v2.go | 16 ++++++++-------- consensus/construct.go | 4 ++-- consensus/debug.go | 7 +++++++ consensus/leader.go | 8 ++++---- consensus/view_change.go | 12 ++++++------ 7 files changed, 40 insertions(+), 21 deletions(-) diff --git a/consensus/checks.go b/consensus/checks.go index e4c9a7084f..fadcfbd04b 100644 --- a/consensus/checks.go +++ b/consensus/checks.go @@ -147,7 +147,7 @@ func (consensus *Consensus) onViewChangeSanityCheck(recvMsg *FBFTMessage) bool { consensus.getLogger().Debug(). Uint64("MsgBlockNum", recvMsg.BlockNum). - Uint64("MyViewChangingID", consensus.GetViewChangingID()). + Uint64("MyViewChangingID", consensus.getViewChangingID()). Uint64("MsgViewChangingID", recvMsg.ViewID). Interface("SendPubKeys", recvMsg.SenderPubkeys). Msg("[onViewChangeSanityCheck]") diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index c43c7fe127..1ce41392cc 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -210,6 +210,13 @@ func (consensus *Consensus) SetIsBackup(isBackup bool) { // Mode returns the mode of consensus func (consensus *Consensus) Mode() Mode { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() + return consensus.mode() +} + +// mode returns the mode of consensus +func (consensus *Consensus) mode() Mode { return consensus.current.Mode() } @@ -254,6 +261,11 @@ func (consensus *Consensus) SetBlockNum(blockNum uint64) { atomic.StoreUint64(&consensus.blockNum, blockNum) } +// SetBlockNum sets the blockNum in consensus object, called at node bootstrap +func (consensus *Consensus) setBlockNum(blockNum uint64) { + atomic.StoreUint64(&consensus.blockNum, blockNum) +} + // ReadSignatureBitmapPayload read the payload for signature and bitmap; offset is the beginning position of reading func (consensus *Consensus) ReadSignatureBitmapPayload(recvPayload []byte, offset int) (*bls_core.Sign, *bls_cosi.Mask, error) { consensus.mutex.RLock() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 1ab33d0bbf..4244cb3732 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -367,7 +367,7 @@ func (consensus *Consensus) syncReadyChan() { func (consensus *Consensus) syncNotReadyChan() { consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncNotReadyChan") - consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) + consensus.setBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) consensus.current.SetMode(Syncing) consensus.getLogger().Info().Msg("[ConsensusMainLoop] Node is OUT OF SYNC") consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() @@ -452,14 +452,14 @@ func (consensus *Consensus) BlockChannel(newBlock *types.Block) { // waitForCommit wait extra 2 seconds for commit phase to finish func (consensus *Consensus) waitForCommit() { - if consensus.Mode() != Normal || consensus.phase.Get() != FBFTCommit { + if consensus.mode() != Normal || consensus.phase.Get() != FBFTCommit { return } // We only need to wait consensus is in normal commit phase utils.Logger().Warn().Str("phase", consensus.phase.String()).Msg("[shutdown] commit phase has to wait") maxWait := time.Now().Add(2 * consensus.BlockPeriod) - for time.Now().Before(maxWait) && consensus.GetConsensusPhase() == "Commit" { + for time.Now().Before(maxWait) && consensus.getConsensusPhase() == "Commit" { utils.Logger().Warn().Msg("[shutdown] wait for consensus finished") time.Sleep(time.Millisecond * 100) } @@ -635,7 +635,7 @@ func (consensus *Consensus) tryCatchup() error { if consensus.BlockVerifier == nil { return errors.New("consensus haven't finished initialization") } - initBN := consensus.BlockNum() + initBN := consensus.getBlockNum() defer consensus.postCatchup(initBN) blks, msgs, err := consensus.getLastMileBlocksAndMsg(initBN) @@ -764,15 +764,15 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg } func (consensus *Consensus) postCatchup(initBN uint64) { - if initBN < consensus.BlockNum() { + if initBN < consensus.getBlockNum() { consensus.getLogger().Info(). Uint64("From", initBN). - Uint64("To", consensus.BlockNum()). + Uint64("To", consensus.getBlockNum()). Msg("[TryCatchup] Caught up!") consensus.switchPhase("TryCatchup", FBFTAnnounce) } // catch up and skip from view change trap - if initBN < consensus.BlockNum() && consensus.isViewChangingMode() { + if initBN < consensus.getBlockNum() && consensus.isViewChangingMode() { consensus.current.SetMode(Normal) consensus.consensusTimeout[timeoutViewChange].Stop() } @@ -833,7 +833,7 @@ func (consensus *Consensus) GenerateVdfAndProof(newBlock *types.Block, vrfBlockN start := time.Now() vdf.Execute() duration := time.Since(start) - consensus.getLogger().Info(). + consensus.GetLogger().Info(). Dur("duration", duration). Msg("[ConsensusMainLoop] VDF computation finished") output := <-outputChannel diff --git a/consensus/construct.go b/consensus/construct.go index 8d9f3838aa..eaa6fd83f1 100644 --- a/consensus/construct.go +++ b/consensus/construct.go @@ -29,8 +29,8 @@ type NetworkMessage struct { func (consensus *Consensus) populateMessageFields( request *msg_pb.ConsensusRequest, blockHash []byte, ) *msg_pb.ConsensusRequest { - request.ViewId = consensus.GetCurBlockViewID() - request.BlockNum = consensus.BlockNum() + request.ViewId = consensus.getCurBlockViewID() + request.BlockNum = consensus.getBlockNum() request.ShardId = consensus.ShardID // 32 byte block hash request.BlockHash = blockHash diff --git a/consensus/debug.go b/consensus/debug.go index da3587710a..cba13cc011 100644 --- a/consensus/debug.go +++ b/consensus/debug.go @@ -2,6 +2,13 @@ package consensus // GetConsensusPhase returns the current phase of the consensus. func (consensus *Consensus) GetConsensusPhase() string { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() + return consensus.getConsensusPhase() +} + +// GetConsensusPhase returns the current phase of the consensus. +func (consensus *Consensus) getConsensusPhase() string { return consensus.phase.String() } diff --git a/consensus/leader.go b/consensus/leader.go index 548ec98c0a..2f7766e19b 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -74,7 +74,7 @@ func (consensus *Consensus) announce(block *types.Block) { } // Construct broadcast p2p message if err := consensus.msgSender.SendWithRetry( - consensus.BlockNum(), msg_pb.MessageType_ANNOUNCE, []nodeconfig.GroupID{ + consensus.getBlockNum(), msg_pb.MessageType_ANNOUNCE, []nodeconfig.GroupID{ nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)), }, p2p.ConstructMessage(msgToSend)); err != nil { consensus.getLogger().Warn(). @@ -95,7 +95,7 @@ func (consensus *Consensus) announce(block *types.Block) { func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { // TODO(audit): make FBFT lookup using map instead of looping through all items. if !consensus.FBFTLog.HasMatchingViewAnnounce( - consensus.BlockNum(), consensus.GetCurBlockViewID(), recvMsg.BlockHash, + consensus.getBlockNum(), consensus.getCurBlockViewID(), recvMsg.BlockHash, ) { consensus.getLogger().Debug(). Uint64("MsgViewID", recvMsg.ViewID). @@ -280,7 +280,7 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { //// Write - End //// Read - Start - viewID := consensus.GetCurBlockViewID() + viewID := consensus.getCurBlockViewID() if consensus.Decider.IsAllSigsCollected() { logger.Info().Msg("[OnCommit] 100% Enough commits received") @@ -315,7 +315,7 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.mutex.Lock() defer consensus.mutex.Unlock() - if viewID == consensus.GetCurBlockViewID() { + if viewID == consensus.getCurBlockViewID() { consensus.finalCommit() } }(viewID) diff --git a/consensus/view_change.go b/consensus/view_change.go index 13acb86ae2..d8b1058a04 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -88,14 +88,14 @@ func (pm *State) SetIsBackup(isBackup bool) { // fallbackNextViewID return the next view ID and duration when there is an exception // to calculate the time-based viewId func (consensus *Consensus) fallbackNextViewID() (uint64, time.Duration) { - diff := int64(consensus.GetViewChangingID() + 1 - consensus.GetCurBlockViewID()) + diff := int64(consensus.getViewChangingID() + 1 - consensus.getCurBlockViewID()) if diff <= 0 { diff = int64(1) } consensus.getLogger().Error(). Int64("diff", diff). Msg("[fallbackNextViewID] use legacy viewID algorithm") - return consensus.GetViewChangingID() + 1, time.Duration(diff * diff * int64(viewChangeDuration)) + return consensus.getViewChangingID() + 1, time.Duration(diff * diff * int64(viewChangeDuration)) } // getNextViewID return the next view ID based on the timestamp @@ -152,7 +152,7 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrapper { gap := 1 - cur := consensus.GetCurBlockViewID() + cur := consensus.getCurBlockViewID() if viewID > cur { gap = int(viewID - cur) } @@ -196,7 +196,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe Str("leaderPubKey", consensus.LeaderPubKey.Bytes.Hex()). Int("gap", gap). Uint64("newViewID", viewID). - Uint64("myCurBlockViewID", consensus.GetCurBlockViewID()). + Uint64("myCurBlockViewID", consensus.getCurBlockViewID()). Msg("[getNextLeaderKey] got leaderPubKey from coinbase") // wasFound, next := consensus.Decider.NthNext(lastLeaderPubKey, gap) // FIXME: rotate leader on harmony nodes only before fully externalization @@ -234,7 +234,7 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - if consensus.disableViewChange || consensus.IsBackup() { + if consensus.disableViewChange || consensus.isBackup { return } @@ -242,7 +242,7 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() - consensus.SetViewChangingID(nextViewID) + consensus.setViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet // we use it this way as in many code we validate the messages From fbd2f7e18643d4451944d4757a5d4de55e7147e3 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sat, 28 Jan 2023 01:11:28 -0300 Subject: [PATCH 126/420] Fix usage of private methods. --- consensus/checks.go | 12 ++++++------ consensus/consensus_service.go | 4 ++-- consensus/consensus_v2.go | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/consensus/checks.go b/consensus/checks.go index fadcfbd04b..28da66ad7b 100644 --- a/consensus/checks.go +++ b/consensus/checks.go @@ -152,7 +152,7 @@ func (consensus *Consensus) onViewChangeSanityCheck(recvMsg *FBFTMessage) bool { Interface("SendPubKeys", recvMsg.SenderPubkeys). Msg("[onViewChangeSanityCheck]") - if consensus.BlockNum() > recvMsg.BlockNum { + if consensus.getBlockNum() > recvMsg.BlockNum { consensus.getLogger().Debug(). Msg("[onViewChange] Message BlockNum Is Low") return false @@ -163,13 +163,13 @@ func (consensus *Consensus) onViewChangeSanityCheck(recvMsg *FBFTMessage) bool { return false } if consensus.isViewChangingMode() && - consensus.GetCurBlockViewID() > recvMsg.ViewID { - consensus.getLogger().Debug().Uint64("curBlockViewID", consensus.GetCurBlockViewID()). + consensus.getCurBlockViewID() > recvMsg.ViewID { + consensus.getLogger().Debug().Uint64("curBlockViewID", consensus.getCurBlockViewID()). Uint64("msgViewID", recvMsg.ViewID). Msg("[onViewChangeSanityCheck] ViewChanging ID Is Low") return false } - if recvMsg.ViewID > consensus.GetViewChangingID() && recvMsg.ViewID-consensus.GetViewChangingID() > MaxViewIDDiff { + if recvMsg.ViewID > consensus.getViewChangingID() && recvMsg.ViewID-consensus.getViewChangingID() > MaxViewIDDiff { consensus.getLogger().Debug(). Msg("[onViewChangeSanityCheck] Received viewID that is MaxViewIDDiff (249) further from the current viewID!") return false @@ -194,9 +194,9 @@ func (consensus *Consensus) onViewChangeSanityCheck(recvMsg *FBFTMessage) bool { // TODO: leo: move the sanity check to p2p message validation func (consensus *Consensus) onNewViewSanityCheck(recvMsg *FBFTMessage) bool { - if recvMsg.ViewID < consensus.GetCurBlockViewID() { + if recvMsg.ViewID < consensus.getCurBlockViewID() { consensus.getLogger().Warn(). - Uint64("LastSuccessfulConsensusViewID", consensus.GetCurBlockViewID()). + Uint64("LastSuccessfulConsensusViewID", consensus.getCurBlockViewID()). Uint64("MsgViewChangingID", recvMsg.ViewID). Msg("[onNewView] ViewID should be larger than the viewID of the last successful consensus") return false diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 1ce41392cc..71582df33a 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -248,9 +248,9 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { Str("leaderKey", consensus.LeaderPubKey.Bytes.Hex()). Msg("[checkViewID] Start consensus timer") return nil - } else if msg.ViewID > consensus.GetCurBlockViewID() { + } else if msg.ViewID > consensus.getCurBlockViewID() { return consensus_engine.ErrViewIDNotMatch - } else if msg.ViewID < consensus.GetCurBlockViewID() { + } else if msg.ViewID < consensus.getCurBlockViewID() { return errors.New("view ID belongs to the past") } return nil diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 4244cb3732..fe6b802b33 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -144,7 +144,7 @@ func (consensus *Consensus) finalCommit() { consensus.getLogger().Info(). Int64("NumCommits", numCommits). Msg("[finalCommit] Finalizing Consensus") - beforeCatchupNum := consensus.BlockNum() + beforeCatchupNum := consensus.getBlockNum() leaderPriKey, err := consensus.getConsensusLeaderPrivateKey() if err != nil { @@ -345,16 +345,16 @@ func (consensus *Consensus) StartChannel() { func (consensus *Consensus) syncReadyChan() { consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncReadyChan") - if consensus.BlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { - consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) - consensus.SetViewIDs(consensus.Blockchain().CurrentHeader().ViewID().Uint64() + 1) + if consensus.getBlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { + consensus.setBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) + consensus.setViewIDs(consensus.Blockchain().CurrentHeader().ViewID().Uint64() + 1) mode := consensus.updateConsensusInformation() consensus.current.SetMode(mode) consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") consensus.consensusTimeout[timeoutConsensus].Start() consensus.getLogger().Info().Str("Mode", mode.String()).Msg("Node is IN SYNC") consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() - } else if consensus.Mode() == Syncing { + } else if consensus.mode() == Syncing { // Corner case where sync is triggered before `onCommitted` and there is a race // for block insertion between consensus and downloader. mode := consensus.updateConsensusInformation() From b2de4d723bb9853849b8703f0d25e5cbd5f84775 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 31 Jan 2023 12:04:33 -0300 Subject: [PATCH 127/420] Removed deadcode, LockedFBFTPhase. --- consensus/consensus.go | 4 ++-- consensus/consensus_fbft.go | 23 ----------------------- consensus/consensus_fbft_test.go | 15 --------------- consensus/consensus_service.go | 2 +- consensus/consensus_v2.go | 15 --------------- 5 files changed, 3 insertions(+), 56 deletions(-) delete mode 100644 consensus/consensus_fbft.go delete mode 100644 consensus/consensus_fbft_test.go diff --git a/consensus/consensus.go b/consensus/consensus.go index 2dbfa52a0c..1d3353f514 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -47,7 +47,7 @@ type Consensus struct { // FBFTLog stores the pbft messages and blocks during FBFT process FBFTLog *FBFTLog // phase: different phase of FBFT protocol: pre-prepare, prepare, commit, finish etc - phase *LockedFBFTPhase + phase FBFTPhase // current indicates what state a node is in current State // isBackup declarative the node is in backup mode @@ -246,7 +246,7 @@ func New( consensus.BlockNumLowChan = make(chan struct{}, 1) // FBFT related consensus.FBFTLog = NewFBFTLog() - consensus.phase = NewLockedFBFTPhase(FBFTAnnounce) + consensus.phase = FBFTAnnounce consensus.current = State{mode: Normal} // FBFT timeout consensus.consensusTimeout = createTimeout() diff --git a/consensus/consensus_fbft.go b/consensus/consensus_fbft.go deleted file mode 100644 index 29c9714d35..0000000000 --- a/consensus/consensus_fbft.go +++ /dev/null @@ -1,23 +0,0 @@ -package consensus - -type LockedFBFTPhase struct { - phase FBFTPhase -} - -func NewLockedFBFTPhase(initialPhrase FBFTPhase) *LockedFBFTPhase { - return &LockedFBFTPhase{ - phase: initialPhrase, - } -} - -func (a *LockedFBFTPhase) Set(phrase FBFTPhase) { - a.phase = phrase -} - -func (a *LockedFBFTPhase) Get() FBFTPhase { - return a.phase -} - -func (a *LockedFBFTPhase) String() string { - return a.Get().String() -} diff --git a/consensus/consensus_fbft_test.go b/consensus/consensus_fbft_test.go deleted file mode 100644 index a84cc3c832..0000000000 --- a/consensus/consensus_fbft_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package consensus - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestLockedFBFTPhase(t *testing.T) { - s := NewLockedFBFTPhase(FBFTAnnounce) - require.Equal(t, FBFTAnnounce, s.Get()) - - s.Set(FBFTCommit) - require.Equal(t, FBFTCommit, s.Get()) -} diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 71582df33a..0e4bb68141 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -554,7 +554,7 @@ func (consensus *Consensus) switchPhase(subject string, desired FBFTPhase) { Str("to:", desired.String()). Str("switchPhase:", subject) - consensus.phase.Set(desired) + consensus.phase = desired } var ( diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index fe6b802b33..3f9a2cfa2d 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -450,21 +450,6 @@ func (consensus *Consensus) BlockChannel(newBlock *types.Block) { } } -// waitForCommit wait extra 2 seconds for commit phase to finish -func (consensus *Consensus) waitForCommit() { - if consensus.mode() != Normal || consensus.phase.Get() != FBFTCommit { - return - } - // We only need to wait consensus is in normal commit phase - utils.Logger().Warn().Str("phase", consensus.phase.String()).Msg("[shutdown] commit phase has to wait") - - maxWait := time.Now().Add(2 * consensus.BlockPeriod) - for time.Now().Before(maxWait) && consensus.getConsensusPhase() == "Commit" { - utils.Logger().Warn().Msg("[shutdown] wait for consensus finished") - time.Sleep(time.Millisecond * 100) - } -} - // LastMileBlockIter is the iterator to iterate over the last mile blocks in consensus cache. // All blocks returned are guaranteed to pass the verification. type LastMileBlockIter struct { From d7b24dd1ba83cea8d0bcb54084c98cb354b6558c Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 31 Jan 2023 12:08:32 -0300 Subject: [PATCH 128/420] Fix review comment. --- consensus/consensus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 1d3353f514..ac15adc15f 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -154,8 +154,8 @@ func (consensus *Consensus) verifyBlock(block *types.Block) error { // thus the blockchain is likely to be up to date. func (consensus *Consensus) BlocksSynchronized() { consensus.mutex.Lock() + defer consensus.mutex.Unlock() consensus.syncReadyChan() - consensus.mutex.Unlock() } // BlocksNotSynchronized lets the main loop know that block is not synchronized From 068f2b2532776468ace5229b2672e7414a03519b Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 31 Jan 2023 12:12:43 -0300 Subject: [PATCH 129/420] Fix review comment. --- consensus/consensus_v2.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 3f9a2cfa2d..6b0d54648d 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -310,9 +310,7 @@ func (consensus *Consensus) Start( case <-stopChan: return case <-ticker.C: - consensus.mutex.Lock() - consensus.tick() - consensus.mutex.Unlock() + consensus.Tick() } } }() @@ -373,6 +371,12 @@ func (consensus *Consensus) syncNotReadyChan() { consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() } +func (consensus *Consensus) Tick() { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + consensus.tick() +} + func (consensus *Consensus) tick() { if !consensus.start && consensus.isInitialLeader { return From 1c1198bf45f77f295d031779fa03a7b571db8869 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 31 Jan 2023 14:29:17 -0300 Subject: [PATCH 130/420] Go mod tidy. --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 883a62f3fe..9506c74697 100644 --- a/go.mod +++ b/go.mod @@ -71,6 +71,7 @@ require ( github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b github.com/ledgerwatch/erigon-lib v0.0.0-20221218022306-0f8fdd40c2db github.com/ledgerwatch/log/v3 v3.6.0 + github.com/libp2p/go-libp2p-core v0.20.1 ) require ( @@ -165,7 +166,6 @@ require ( github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect - github.com/libp2p/go-libp2p-core v0.20.1 // indirect github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect github.com/libp2p/go-msgio v0.2.0 // indirect From caf7f54d0d39b3ae55cde9edfc7c7f4aafbe886c Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 31 Jan 2023 18:03:38 -0300 Subject: [PATCH 131/420] Set to EpochTBD. --- consensus/consensus_test.go | 2 +- internal/params/config.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/consensus_test.go b/consensus/consensus_test.go index 7d74d8815a..41fe5b127f 100644 --- a/consensus/consensus_test.go +++ b/consensus/consensus_test.go @@ -39,7 +39,7 @@ func TestConsensusInitialization(t *testing.T) { // FBFTLog assert.Equal(t, fbtLog, consensus.FBFTLog) - assert.Equal(t, FBFTAnnounce, consensus.phase.Get()) + assert.Equal(t, FBFTAnnounce, consensus.phase) // State / consensus.current assert.Equal(t, state.mode, consensus.current.mode) diff --git a/internal/params/config.go b/internal/params/config.go index 8ee199d22a..b50aa35532 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -271,7 +271,7 @@ var ( SlotsLimitedEpoch: EpochTBD, // epoch to enable HIP-16 CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, - LeaderRotationEpoch: big.NewInt(4), + LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 5, FeeCollectEpoch: big.NewInt(5), } From 2ec0cb736af556b0cae9386dbfe281c70f26a4f9 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 31 Jan 2023 19:27:43 -0300 Subject: [PATCH 132/420] Fix tests. --- consensus/view_change_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/consensus/view_change_test.go b/consensus/view_change_test.go index fc80b6ccfe..2d149b6b7b 100644 --- a/consensus/view_change_test.go +++ b/consensus/view_change_test.go @@ -43,7 +43,7 @@ func TestPhaseSwitching(t *testing.T) { _, _, consensus, _, err := GenerateConsensusForTesting() assert.NoError(t, err) - assert.Equal(t, FBFTAnnounce, consensus.phase.Get()) // It's a new consensus, we should be at the FBFTAnnounce phase. + assert.Equal(t, FBFTAnnounce, consensus.phase) // It's a new consensus, we should be at the FBFTAnnounce phase. switches := []phaseSwitch{ {start: FBFTAnnounce, end: FBFTPrepare}, @@ -73,10 +73,10 @@ func TestPhaseSwitching(t *testing.T) { func testPhaseGroupSwitching(t *testing.T, consensus *Consensus, phases []FBFTPhase, startPhase FBFTPhase, desiredPhase FBFTPhase) { for range phases { consensus.switchPhase("test", desiredPhase) - assert.Equal(t, desiredPhase, consensus.phase.Get()) + assert.Equal(t, desiredPhase, consensus.phase) } - assert.Equal(t, desiredPhase, consensus.phase.Get()) + assert.Equal(t, desiredPhase, consensus.phase) return } From 881d8d70507b8829167383662e6ad0ceef936a02 Mon Sep 17 00:00:00 2001 From: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Date: Thu, 16 Feb 2023 02:57:49 +0000 Subject: [PATCH 133/420] [core] fix state handling of self destruct If a contract self destructs to self and then receives funds within the same transaction, it is possible for its stale state to be saved. This change removes that possibility by checking for deleted state objects before returning them. --- core/state/statedb.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index d571d3372d..0a7cd3b564 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -542,7 +542,10 @@ func (db *DB) createObject(addr common.Address) (newobj, prev *Object) { db.journal.append(resetObjectChange{prev: prev}) } db.setStateObject(newobj) - return newobj, prev + if prev != nil && !prev.deleted { + return newobj, prev + } + return newobj, nil } // CreateAccount explicitly creates a state object. If a state object with the address From 2b9b2f580b06626e56dc543d2f6f0309db19f9a4 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sun, 19 Feb 2023 15:43:04 +0000 Subject: [PATCH 134/420] Fixed race error. --- api/service/explorer/schema.go | 2 +- api/service/explorer/storage.go | 54 ++++++++++++++++++++++++++++++--- go.mod | 2 +- go.sum | 4 +-- scripts/go_executable_build.sh | 2 +- 5 files changed, 54 insertions(+), 10 deletions(-) diff --git a/api/service/explorer/schema.go b/api/service/explorer/schema.go index 09848573d0..3104d75dc5 100644 --- a/api/service/explorer/schema.go +++ b/api/service/explorer/schema.go @@ -41,7 +41,7 @@ func readCheckpointBitmap(db databaseReader) (*roaring64.Bitmap, error) { } // writeCheckpointBitmap write explorer checkpoint bitmap to storage -func writeCheckpointBitmap(db databaseWriter, rb *roaring64.Bitmap) error { +func writeCheckpointBitmap(db databaseWriter, rb Bitmap) error { bitmapByte, err := rb.MarshalBinary() if err != nil { return err diff --git a/api/service/explorer/storage.go b/api/service/explorer/storage.go index 72d0281805..ebb4e2577b 100644 --- a/api/service/explorer/storage.go +++ b/api/service/explorer/storage.go @@ -34,11 +34,54 @@ const ( // explorer db is doing migration and unavailable var ErrExplorerNotReady = errors.New("explorer db not ready") +type Bitmap interface { + Clone() *roaring64.Bitmap + MarshalBinary() ([]byte, error) + CheckedAdd(x uint64) bool + Contains(x uint64) bool +} + +type ThreadSafeBitmap struct { + b Bitmap + mu sync.Mutex +} + +func (a *ThreadSafeBitmap) Clone() *roaring64.Bitmap { + a.mu.Lock() + defer a.mu.Unlock() + return a.b.Clone() +} + +func (a *ThreadSafeBitmap) CheckedAdd(x uint64) bool { + a.mu.Lock() + defer a.mu.Unlock() + return a.b.CheckedAdd(x) +} + +func (a *ThreadSafeBitmap) Contains(x uint64) bool { + a.mu.Lock() + defer a.mu.Unlock() + return a.b.Contains(x) +} + +func (a *ThreadSafeBitmap) MarshalBinary() ([]byte, error) { + a.mu.Lock() + defer a.mu.Unlock() + return a.b.MarshalBinary() +} + +func NewThreadSafeBitmap(bitmap Bitmap) Bitmap { + return &ThreadSafeBitmap{ + b: bitmap, + mu: sync.Mutex{}, + } +} + type ( storage struct { db database bc core.BlockChain - rb *roaring64.Bitmap + rb Bitmap //*roaring64.Bitmap // TODO: optimize this with priority queue tm *taskManager @@ -89,11 +132,12 @@ func newStorage(hc *harmonyconfig.HarmonyConfig, bc core.BlockChain, dbPath stri return nil, err } + safeBitmap := NewThreadSafeBitmap(bitmap) return &storage{ db: db, bc: bc, - rb: bitmap, - tm: newTaskManager(bitmap), + rb: safeBitmap, + tm: newTaskManager(safeBitmap), resultC: make(chan blockResult, numWorker), resultT: make(chan *traceResult, numWorker), available: abool.New(), @@ -211,14 +255,14 @@ type taskManager struct { blocksLP []*types.Block // blocks with low priorities lock sync.Mutex - rb *roaring64.Bitmap + rb Bitmap rbChangedCount int C chan struct{} T chan *traceResult } -func newTaskManager(bitmap *roaring64.Bitmap) *taskManager { +func newTaskManager(bitmap Bitmap) *taskManager { return &taskManager{ rb: bitmap, C: make(chan struct{}, numWorker), diff --git a/go.mod b/go.mod index 9506c74697..561a021ce8 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/harmony-one/harmony go 1.19 require ( - github.com/RoaringBitmap/roaring v1.2.1 + github.com/RoaringBitmap/roaring v1.2.3 github.com/VictoriaMetrics/fastcache v1.5.7 github.com/Workiva/go-datastructures v1.0.50 github.com/allegro/bigcache v1.2.1 diff --git a/go.sum b/go.sum index 0bcfa714fb..48d1761c4a 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us= github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= -github.com/RoaringBitmap/roaring v1.2.1 h1:58/LJlg/81wfEHd5L9qsHduznOIhyv4qb1yWcSvVq9A= -github.com/RoaringBitmap/roaring v1.2.1/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= +github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= +github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw= diff --git a/scripts/go_executable_build.sh b/scripts/go_executable_build.sh index 2c1cc99f0e..6331aa5225 100755 --- a/scripts/go_executable_build.sh +++ b/scripts/go_executable_build.sh @@ -21,7 +21,7 @@ VERBOSE= if [ "$(uname -s)" == "Darwin" ]; then GO_GCFLAGS="" else - GO_GCFLAGS="all=-c 2" + GO_GCFLAGS="" fi DEBUG=false STATIC=true From a6b3f890e4925439720e018c7387ec6e2a36d4bc Mon Sep 17 00:00:00 2001 From: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Date: Thu, 23 Feb 2023 06:19:04 +0000 Subject: [PATCH 135/420] rpc: add configurable http and `eth_call` timeout --- cmd/harmony/config_migrations.go | 17 +++++ cmd/harmony/default.go | 6 +- cmd/harmony/flags.go | 39 +++++++++++- cmd/harmony/flags_test.go | 58 +++++++++++++++++ internal/configs/harmony/harmony.go | 64 +++++++++++++++++++ internal/configs/harmony/harmony_test.go | 81 ++++++++++++++++++++++++ internal/configs/node/config.go | 7 ++ internal/configs/node/network.go | 6 ++ rosetta/rosetta.go | 5 +- rosetta/services/call_service.go | 10 ++- rpc/contract.go | 22 +++++-- rpc/rpc.go | 25 +++++--- 12 files changed, 320 insertions(+), 20 deletions(-) create mode 100644 internal/configs/harmony/harmony_test.go diff --git a/cmd/harmony/config_migrations.go b/cmd/harmony/config_migrations.go index 7c9420a0be..b3da9ec2ba 100644 --- a/cmd/harmony/config_migrations.go +++ b/cmd/harmony/config_migrations.go @@ -340,6 +340,23 @@ func init() { return confTree } + migrations["2.5.12"] = func(confTree *toml.Tree) *toml.Tree { + if confTree.Get("HTTP.ReadTimeout") == nil { + confTree.Set("HTTP.ReadTimeout", defaultConfig.HTTP.ReadTimeout) + } + if confTree.Get("HTTP.WriteTimeout") == nil { + confTree.Set("HTTP.WriteTimeout", defaultConfig.HTTP.WriteTimeout) + } + if confTree.Get("HTTP.IdleTimeout") == nil { + confTree.Set("HTTP.IdleTimeout", defaultConfig.HTTP.IdleTimeout) + } + if confTree.Get("RPCOpt.EvmCallTimeout") == nil { + confTree.Set("RPCOpt.EvmCallTimeout", defaultConfig.RPCOpt.EvmCallTimeout) + } + confTree.Set("Version", "2.5.13") + return confTree + } + // check that the latest version here is the same as in default.go largestKey := getNextVersion(migrations) if largestKey != tomlConfigVersion { diff --git a/cmd/harmony/default.go b/cmd/harmony/default.go index 3835c49210..95e05b29cc 100644 --- a/cmd/harmony/default.go +++ b/cmd/harmony/default.go @@ -5,7 +5,7 @@ import ( nodeconfig "github.com/harmony-one/harmony/internal/configs/node" ) -const tomlConfigVersion = "2.5.12" +const tomlConfigVersion = "2.5.13" const ( defNetworkType = nodeconfig.Mainnet @@ -44,6 +44,9 @@ var defaultConfig = harmonyconfig.HarmonyConfig{ Port: nodeconfig.DefaultRPCPort, AuthPort: nodeconfig.DefaultAuthRPCPort, RosettaPort: nodeconfig.DefaultRosettaPort, + ReadTimeout: nodeconfig.DefaultHTTPTimeoutRead, + WriteTimeout: nodeconfig.DefaultHTTPTimeoutWrite, + IdleTimeout: nodeconfig.DefaultHTTPTimeoutIdle, }, WS: harmonyconfig.WsConfig{ Enabled: true, @@ -59,6 +62,7 @@ var defaultConfig = harmonyconfig.HarmonyConfig{ RpcFilterFile: "./.hmy/rpc_filter.txt", RateLimterEnabled: true, RequestsPerSecond: nodeconfig.DefaultRPCRateLimit, + EvmCallTimeout: nodeconfig.DefaultEvmCallTimeout, }, BLSKeys: harmonyconfig.BlsConfig{ KeyDir: "./.hmy/blskeys", diff --git a/cmd/harmony/flags.go b/cmd/harmony/flags.go index 2ffceb6e5b..8a2799ce28 100644 --- a/cmd/harmony/flags.go +++ b/cmd/harmony/flags.go @@ -75,6 +75,9 @@ var ( httpPortFlag, httpAuthPortFlag, httpRosettaPortFlag, + httpReadTimeoutFlag, + httpWriteTimeoutFlag, + httpIdleTimeoutFlag, } wsFlags = []cli.Flag{ @@ -92,6 +95,7 @@ var ( rpcFilterFileFlag, rpcRateLimiterEnabledFlag, rpcRateLimitFlag, + rpcEvmCallTimeoutFlag, } blsFlags = append(newBLSFlags, legacyBLSFlags...) @@ -695,6 +699,21 @@ var ( Usage: "rosetta port to listen for HTTP requests", DefValue: defaultConfig.HTTP.RosettaPort, } + httpReadTimeoutFlag = cli.StringFlag{ + Name: "http.timeout.read", + Usage: "maximum duration to read the entire request, including the body", + DefValue: defaultConfig.HTTP.ReadTimeout, + } + httpWriteTimeoutFlag = cli.StringFlag{ + Name: "http.timeout.write", + Usage: "maximum duration before timing out writes of the response", + DefValue: defaultConfig.HTTP.WriteTimeout, + } + httpIdleTimeoutFlag = cli.StringFlag{ + Name: "http.timeout.idle", + Usage: "maximum amount of time to wait for the next request when keep-alives are enabled", + DefValue: defaultConfig.HTTP.IdleTimeout, + } ) func applyHTTPFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { @@ -732,6 +751,16 @@ func applyHTTPFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { config.HTTP.Enabled = true } + if cli.IsFlagChanged(cmd, httpReadTimeoutFlag) { + config.HTTP.ReadTimeout = cli.GetStringFlagValue(cmd, httpReadTimeoutFlag) + } + if cli.IsFlagChanged(cmd, httpWriteTimeoutFlag) { + config.HTTP.WriteTimeout = cli.GetStringFlagValue(cmd, httpWriteTimeoutFlag) + } + if cli.IsFlagChanged(cmd, httpIdleTimeoutFlag) { + config.HTTP.IdleTimeout = cli.GetStringFlagValue(cmd, httpIdleTimeoutFlag) + } + } // ws flags @@ -821,6 +850,12 @@ var ( Usage: "the number of requests per second for RPCs", DefValue: defaultConfig.RPCOpt.RequestsPerSecond, } + + rpcEvmCallTimeoutFlag = cli.StringFlag{ + Name: "rpc.evm-call-timeout", + Usage: "timeout for evm execution (eth_call); 0 means infinite timeout", + DefValue: defaultConfig.RPCOpt.EvmCallTimeout, + } ) func applyRPCOptFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { @@ -845,7 +880,9 @@ func applyRPCOptFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { if cli.IsFlagChanged(cmd, rpcRateLimitFlag) { config.RPCOpt.RequestsPerSecond = cli.GetIntFlagValue(cmd, rpcRateLimitFlag) } - + if cli.IsFlagChanged(cmd, rpcEvmCallTimeoutFlag) { + config.RPCOpt.EvmCallTimeout = cli.GetStringFlagValue(cmd, rpcEvmCallTimeoutFlag) + } } // bls flags diff --git a/cmd/harmony/flags_test.go b/cmd/harmony/flags_test.go index 15ba9aaf08..2015188ed4 100644 --- a/cmd/harmony/flags_test.go +++ b/cmd/harmony/flags_test.go @@ -77,6 +77,9 @@ func TestHarmonyFlags(t *testing.T) { AuthPort: 9501, RosettaEnabled: false, RosettaPort: 9700, + ReadTimeout: defaultConfig.HTTP.ReadTimeout, + WriteTimeout: defaultConfig.HTTP.WriteTimeout, + IdleTimeout: defaultConfig.HTTP.IdleTimeout, }, RPCOpt: harmonyconfig.RpcOptConfig{ DebugEnabled: false, @@ -86,6 +89,7 @@ func TestHarmonyFlags(t *testing.T) { RpcFilterFile: "./.hmy/rpc_filter.txt", RateLimterEnabled: true, RequestsPerSecond: 1000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, }, WS: harmonyconfig.WsConfig{ Enabled: true, @@ -531,6 +535,9 @@ func TestRPCFlags(t *testing.T) { Port: defaultConfig.HTTP.Port, AuthPort: defaultConfig.HTTP.AuthPort, RosettaPort: defaultConfig.HTTP.RosettaPort, + ReadTimeout: defaultConfig.HTTP.ReadTimeout, + WriteTimeout: defaultConfig.HTTP.WriteTimeout, + IdleTimeout: defaultConfig.HTTP.IdleTimeout, }, }, { @@ -542,6 +549,9 @@ func TestRPCFlags(t *testing.T) { Port: 9001, AuthPort: defaultConfig.HTTP.AuthPort, RosettaPort: defaultConfig.HTTP.RosettaPort, + ReadTimeout: defaultConfig.HTTP.ReadTimeout, + WriteTimeout: defaultConfig.HTTP.WriteTimeout, + IdleTimeout: defaultConfig.HTTP.IdleTimeout, }, }, { @@ -553,6 +563,9 @@ func TestRPCFlags(t *testing.T) { Port: defaultConfig.HTTP.Port, AuthPort: 9001, RosettaPort: defaultConfig.HTTP.RosettaPort, + ReadTimeout: defaultConfig.HTTP.ReadTimeout, + WriteTimeout: defaultConfig.HTTP.WriteTimeout, + IdleTimeout: defaultConfig.HTTP.IdleTimeout, }, }, { @@ -564,6 +577,9 @@ func TestRPCFlags(t *testing.T) { Port: 9001, AuthPort: defaultConfig.HTTP.AuthPort, RosettaPort: 10001, + ReadTimeout: defaultConfig.HTTP.ReadTimeout, + WriteTimeout: defaultConfig.HTTP.WriteTimeout, + IdleTimeout: defaultConfig.HTTP.IdleTimeout, }, }, { @@ -575,6 +591,9 @@ func TestRPCFlags(t *testing.T) { Port: defaultConfig.HTTP.Port, AuthPort: defaultConfig.HTTP.AuthPort, RosettaPort: 10001, + ReadTimeout: defaultConfig.HTTP.ReadTimeout, + WriteTimeout: defaultConfig.HTTP.WriteTimeout, + IdleTimeout: defaultConfig.HTTP.IdleTimeout, }, }, { @@ -586,6 +605,23 @@ func TestRPCFlags(t *testing.T) { Port: 9501, AuthPort: 9502, RosettaPort: 9701, + ReadTimeout: defaultConfig.HTTP.ReadTimeout, + WriteTimeout: defaultConfig.HTTP.WriteTimeout, + IdleTimeout: defaultConfig.HTTP.IdleTimeout, + }, + }, + { + args: []string{"--http.timeout.read", "10s", "--http.timeout.write", "20s", "--http.timeout.idle", "30s"}, + expConfig: harmonyconfig.HttpConfig{ + Enabled: true, + RosettaEnabled: false, + IP: defaultConfig.HTTP.IP, + Port: defaultConfig.HTTP.Port, + AuthPort: defaultConfig.HTTP.AuthPort, + RosettaPort: defaultConfig.HTTP.RosettaPort, + ReadTimeout: "10s", + WriteTimeout: "20s", + IdleTimeout: "30s", }, }, } @@ -699,6 +735,7 @@ func TestRPCOptFlags(t *testing.T) { RpcFilterFile: "./.hmy/rpc_filter.txt", RateLimterEnabled: true, RequestsPerSecond: 1000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, }, }, @@ -712,6 +749,7 @@ func TestRPCOptFlags(t *testing.T) { RpcFilterFile: "./.hmy/rpc_filter.txt", RateLimterEnabled: true, RequestsPerSecond: 1000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, }, }, @@ -725,6 +763,7 @@ func TestRPCOptFlags(t *testing.T) { RpcFilterFile: "./.hmy/rpc_filter.txt", RateLimterEnabled: true, RequestsPerSecond: 1000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, }, }, @@ -738,6 +777,7 @@ func TestRPCOptFlags(t *testing.T) { RpcFilterFile: "./.hmy/rpc_filter.txt", RateLimterEnabled: true, RequestsPerSecond: 1000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, }, }, @@ -751,6 +791,7 @@ func TestRPCOptFlags(t *testing.T) { RpcFilterFile: "./rmf.toml", RateLimterEnabled: true, RequestsPerSecond: 1000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, }, }, @@ -764,6 +805,7 @@ func TestRPCOptFlags(t *testing.T) { RpcFilterFile: "./.hmy/rpc_filter.txt", RateLimterEnabled: true, RequestsPerSecond: 1000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, }, }, @@ -777,6 +819,7 @@ func TestRPCOptFlags(t *testing.T) { RpcFilterFile: "./.hmy/rpc_filter.txt", RateLimterEnabled: true, RequestsPerSecond: 2000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, }, }, @@ -790,6 +833,21 @@ func TestRPCOptFlags(t *testing.T) { RpcFilterFile: "./.hmy/rpc_filter.txt", RateLimterEnabled: false, RequestsPerSecond: 2000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, + }, + }, + + { + args: []string{"--rpc.evm-call-timeout", "10s"}, + expConfig: harmonyconfig.RpcOptConfig{ + DebugEnabled: false, + EthRPCsEnabled: true, + StakingRPCsEnabled: true, + LegacyRPCsEnabled: true, + RpcFilterFile: "./.hmy/rpc_filter.txt", + RateLimterEnabled: true, + RequestsPerSecond: 1000, + EvmCallTimeout: "10s", }, }, } diff --git a/internal/configs/harmony/harmony.go b/internal/configs/harmony/harmony.go index 67c29f820f..d4e8df4b00 100644 --- a/internal/configs/harmony/harmony.go +++ b/internal/configs/harmony/harmony.go @@ -3,6 +3,10 @@ package harmony import ( "reflect" "strings" + "time" + + nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + "github.com/harmony-one/harmony/internal/utils" ) // HarmonyConfig contains all the configs user can set for running harmony binary. Served as the bridge @@ -32,6 +36,62 @@ type HarmonyConfig struct { ShardData ShardDataConfig } +func (hc HarmonyConfig) ToRPCServerConfig() nodeconfig.RPCServerConfig { + readTimeout, err := time.ParseDuration(hc.HTTP.ReadTimeout) + if err != nil { + readTimeout, _ = time.ParseDuration(nodeconfig.DefaultHTTPTimeoutRead) + utils.Logger().Warn(). + Str("provided", hc.HTTP.ReadTimeout). + Dur("updated", readTimeout). + Msg("Sanitizing invalid http read timeout") + } + writeTimeout, err := time.ParseDuration(hc.HTTP.WriteTimeout) + if err != nil { + writeTimeout, _ = time.ParseDuration(nodeconfig.DefaultHTTPTimeoutWrite) + utils.Logger().Warn(). + Str("provided", hc.HTTP.WriteTimeout). + Dur("updated", writeTimeout). + Msg("Sanitizing invalid http write timeout") + } + idleTimeout, err := time.ParseDuration(hc.HTTP.IdleTimeout) + if err != nil { + idleTimeout, _ = time.ParseDuration(nodeconfig.DefaultHTTPTimeoutIdle) + utils.Logger().Warn(). + Str("provided", hc.HTTP.IdleTimeout). + Dur("updated", idleTimeout). + Msg("Sanitizing invalid http idle timeout") + } + evmCallTimeout, err := time.ParseDuration(hc.RPCOpt.EvmCallTimeout) + if err != nil { + evmCallTimeout, _ = time.ParseDuration(nodeconfig.DefaultEvmCallTimeout) + utils.Logger().Warn(). + Str("provided", hc.RPCOpt.EvmCallTimeout). + Dur("updated", evmCallTimeout). + Msg("Sanitizing invalid evm_call timeout") + } + return nodeconfig.RPCServerConfig{ + HTTPEnabled: hc.HTTP.Enabled, + HTTPIp: hc.HTTP.IP, + HTTPPort: hc.HTTP.Port, + HTTPAuthPort: hc.HTTP.AuthPort, + HTTPTimeoutRead: readTimeout, + HTTPTimeoutWrite: writeTimeout, + HTTPTimeoutIdle: idleTimeout, + WSEnabled: hc.WS.Enabled, + WSIp: hc.WS.IP, + WSPort: hc.WS.Port, + WSAuthPort: hc.WS.AuthPort, + DebugEnabled: hc.RPCOpt.DebugEnabled, + EthRPCsEnabled: hc.RPCOpt.EthRPCsEnabled, + StakingRPCsEnabled: hc.RPCOpt.StakingRPCsEnabled, + LegacyRPCsEnabled: hc.RPCOpt.LegacyRPCsEnabled, + RpcFilterFile: hc.RPCOpt.RpcFilterFile, + RateLimiterEnabled: hc.RPCOpt.RateLimterEnabled, + RequestsPerSecond: hc.RPCOpt.RequestsPerSecond, + EvmCallTimeout: evmCallTimeout, + } +} + type DnsSync struct { Port int // replaces: Network.DNSSyncPort Zone string // replaces: Network.DNSZone @@ -180,6 +240,9 @@ type HttpConfig struct { AuthPort int RosettaEnabled bool RosettaPort int + ReadTimeout string + WriteTimeout string + IdleTimeout string } type WsConfig struct { @@ -197,6 +260,7 @@ type RpcOptConfig struct { RpcFilterFile string // Define filters to enable/disable RPC exposure RateLimterEnabled bool // Enable Rate limiter for RPC RequestsPerSecond int // for RPC rate limiter + EvmCallTimeout string // Timeout for eth_call } type DevnetConfig struct { diff --git a/internal/configs/harmony/harmony_test.go b/internal/configs/harmony/harmony_test.go new file mode 100644 index 0000000000..fef7cac9df --- /dev/null +++ b/internal/configs/harmony/harmony_test.go @@ -0,0 +1,81 @@ +package harmony + +import ( + "fmt" + "testing" + "time" + + nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + "github.com/stretchr/testify/assert" +) + +func TestToRPCServerConfig(t *testing.T) { + tests := []struct { + input HarmonyConfig + output nodeconfig.RPCServerConfig + }{ + { + input: HarmonyConfig{ + HTTP: HttpConfig{ + Enabled: true, + RosettaEnabled: false, + IP: "127.0.0.1", + Port: nodeconfig.DefaultRPCPort, + AuthPort: nodeconfig.DefaultAuthRPCPort, + RosettaPort: nodeconfig.DefaultRosettaPort, + ReadTimeout: "-1", + WriteTimeout: "-2", + IdleTimeout: "-3", + }, + WS: WsConfig{ + Enabled: true, + IP: "127.0.0.1", + Port: nodeconfig.DefaultWSPort, + AuthPort: nodeconfig.DefaultAuthWSPort, + }, + RPCOpt: RpcOptConfig{ + DebugEnabled: false, + EthRPCsEnabled: true, + StakingRPCsEnabled: true, + LegacyRPCsEnabled: true, + RpcFilterFile: "./.hmy/rpc_filter.txt", + RateLimterEnabled: true, + RequestsPerSecond: nodeconfig.DefaultRPCRateLimit, + EvmCallTimeout: "-4", + }, + }, + output: nodeconfig.RPCServerConfig{ + HTTPEnabled: true, + HTTPIp: "127.0.0.1", + HTTPPort: nodeconfig.DefaultRPCPort, + HTTPAuthPort: nodeconfig.DefaultAuthRPCPort, + HTTPTimeoutRead: 30 * time.Second, + HTTPTimeoutWrite: 30 * time.Second, + HTTPTimeoutIdle: 120 * time.Second, + WSEnabled: true, + WSIp: "127.0.0.1", + WSPort: nodeconfig.DefaultWSPort, + WSAuthPort: nodeconfig.DefaultAuthWSPort, + DebugEnabled: false, + EthRPCsEnabled: true, + StakingRPCsEnabled: true, + LegacyRPCsEnabled: true, + RpcFilterFile: "./.hmy/rpc_filter.txt", + RateLimiterEnabled: true, + RequestsPerSecond: nodeconfig.DefaultRPCRateLimit, + EvmCallTimeout: 5 * time.Second, + }, + }, + } + for i, tt := range tests { + assertObject := assert.New(t) + name := fmt.Sprintf("TestToRPCServerConfig: #%d", i) + t.Run(name, func(t *testing.T) { + assertObject.Equal( + tt.input.ToRPCServerConfig(), + tt.output, + name, + ) + }) + } +} diff --git a/internal/configs/node/config.go b/internal/configs/node/config.go index 9a0e950ecc..9f681fca9f 100644 --- a/internal/configs/node/config.go +++ b/internal/configs/node/config.go @@ -8,6 +8,7 @@ import ( "math/big" "strings" "sync" + "time" bls_core "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/crypto/bls" @@ -115,6 +116,10 @@ type RPCServerConfig struct { HTTPPort int HTTPAuthPort int + HTTPTimeoutRead time.Duration + HTTPTimeoutWrite time.Duration + HTTPTimeoutIdle time.Duration + WSEnabled bool WSIp string WSPort int @@ -130,6 +135,8 @@ type RPCServerConfig struct { RateLimiterEnabled bool RequestsPerSecond int + + EvmCallTimeout time.Duration } // RosettaServerConfig is the config for the rosetta server diff --git a/internal/configs/node/network.go b/internal/configs/node/network.go index 8b15d3359c..03f1472125 100644 --- a/internal/configs/node/network.go +++ b/internal/configs/node/network.go @@ -51,6 +51,12 @@ const ( DefaultAuthRPCPort = 9501 // DefaultRosettaPort is the default rosetta port. The actual port used is 9000+700 DefaultRosettaPort = 9700 + // DefaultHTTP timeouts - read, write, and idle + DefaultHTTPTimeoutRead = "30s" + DefaultHTTPTimeoutWrite = "30s" + DefaultHTTPTimeoutIdle = "120s" + // DefaultEvmCallTimeout is the default timeout for evm call + DefaultEvmCallTimeout = "5s" // DefaultWSPort is the default port for web socket endpoint. The actual port used is DefaultWSPort = 9800 // DefaultAuthWSPort is the default port for web socket auth endpoint. The actual port used is diff --git a/rosetta/rosetta.go b/rosetta/rosetta.go index 860c2d0d66..5ff222a65b 100644 --- a/rosetta/rosetta.go +++ b/rosetta/rosetta.go @@ -84,7 +84,10 @@ func getRouter(asserter *asserter.Asserter, hmy *hmy.Harmony, limiterEnable bool server.NewMempoolAPIController(services.NewMempoolAPI(hmy), asserter), server.NewNetworkAPIController(services.NewNetworkAPI(hmy), asserter), server.NewConstructionAPIController(services.NewConstructionAPI(hmy), asserter), - server.NewCallAPIController(services.NewCallAPIService(hmy, limiterEnable, rateLimit), asserter), + server.NewCallAPIController( + services.NewCallAPIService(hmy, limiterEnable, rateLimit, hmy.NodeAPI.GetConfig().NodeConfig.RPCServer.EvmCallTimeout), + asserter, + ), server.NewEventsAPIController(services.NewEventAPI(hmy), asserter), server.NewSearchAPIController(services.NewSearchAPI(hmy), asserter), ) diff --git a/rosetta/services/call_service.go b/rosetta/services/call_service.go index 46f528d7b9..9d26bab282 100644 --- a/rosetta/services/call_service.go +++ b/rosetta/services/call_service.go @@ -3,6 +3,7 @@ package services import ( "context" "encoding/json" + "time" "github.com/coinbase/rosetta-sdk-go/server" "github.com/coinbase/rosetta-sdk-go/types" @@ -82,10 +83,15 @@ func (c *CallAPIService) Call( } -func NewCallAPIService(hmy *hmy.Harmony, limiterEnable bool, rateLimit int) server.CallAPIServicer { +func NewCallAPIService( + hmy *hmy.Harmony, + limiterEnable bool, + rateLimit int, + evmCallTimeout time.Duration, +) server.CallAPIServicer { return &CallAPIService{ hmy: hmy, - publicContractAPI: rpc2.NewPublicContractAPI(hmy, rpc2.V2, limiterEnable, rateLimit), + publicContractAPI: rpc2.NewPublicContractAPI(hmy, rpc2.V2, limiterEnable, rateLimit, evmCallTimeout), publicStakingAPI: rpc2.NewPublicStakingAPI(hmy, rpc2.V2), publicBlockChainAPI: rpc2.NewPublicBlockchainAPI(hmy, rpc2.V2, limiterEnable, rateLimit), } diff --git a/rpc/contract.go b/rpc/contract.go index 337bea7cde..abcb4f9418 100644 --- a/rpc/contract.go +++ b/rpc/contract.go @@ -31,11 +31,18 @@ type PublicContractService struct { hmy *hmy.Harmony version Version // TEMP SOLUTION to rpc node spamming issue - limiterCall *rate.Limiter + limiterCall *rate.Limiter + evmCallTimeout time.Duration } // NewPublicContractAPI creates a new API for the RPC interface -func NewPublicContractAPI(hmy *hmy.Harmony, version Version, limiterEnable bool, limit int) rpc.API { +func NewPublicContractAPI( + hmy *hmy.Harmony, + version Version, + limiterEnable bool, + limit int, + evmCallTimeout time.Duration, +) rpc.API { var limiter *rate.Limiter if limiterEnable { limiter = rate.NewLimiter(rate.Limit(limit), limit) @@ -44,8 +51,13 @@ func NewPublicContractAPI(hmy *hmy.Harmony, version Version, limiterEnable bool, return rpc.API{ Namespace: version.Namespace(), Version: APIVersion, - Service: &PublicContractService{hmy, version, limiter}, - Public: true, + Service: &PublicContractService{ + hmy: hmy, + version: version, + limiterCall: limiter, + evmCallTimeout: evmCallTimeout, + }, + Public: true, } } @@ -80,7 +92,7 @@ func (s *PublicContractService) Call( } // Execute call - result, err := DoEVMCall(ctx, s.hmy, args, blockNrOrHash, CallTimeout) + result, err := DoEVMCall(ctx, s.hmy, args, blockNrOrHash, s.evmCallTimeout) if err != nil { return nil, err } diff --git a/rpc/rpc.go b/rpc/rpc.go index 2c94f0a5e5..1bae7367de 100644 --- a/rpc/rpc.go +++ b/rpc/rpc.go @@ -58,9 +58,9 @@ var ( wsEndpoint = "" wsAuthEndpoint = "" httpVirtualHosts = []string{"*"} - httpTimeouts = rpc.DefaultHTTPTimeouts - httpOrigins = []string{"*"} - wsOrigins = []string{"*"} + // httpTimeouts = rpc.DefaultHTTPTimeouts + httpOrigins = []string{"*"} + wsOrigins = []string{"*"} ) // Version of the RPC @@ -86,13 +86,18 @@ func StartServers(hmy *hmy.Harmony, apis []rpc.API, config nodeconfig.RPCServerC rmf.ExposeAll() } if config.HTTPEnabled { + timeouts := rpc.HTTPTimeouts{ + ReadTimeout: config.HTTPTimeoutRead, + WriteTimeout: config.HTTPTimeoutWrite, + IdleTimeout: config.HTTPTimeoutIdle, + } httpEndpoint = fmt.Sprintf("%v:%v", config.HTTPIp, config.HTTPPort) - if err := startHTTP(apis, &rmf); err != nil { + if err := startHTTP(apis, &rmf, timeouts); err != nil { return err } httpAuthEndpoint = fmt.Sprintf("%v:%v", config.HTTPIp, config.HTTPAuthPort) - if err := startAuthHTTP(authApis, &rmf); err != nil { + if err := startAuthHTTP(authApis, &rmf, timeouts); err != nil { return err } } @@ -158,8 +163,8 @@ func getAPIs(hmy *hmy.Harmony, config nodeconfig.RPCServerConfig) []rpc.API { NewPublicHarmonyAPI(hmy, V2), NewPublicBlockchainAPI(hmy, V1, config.RateLimiterEnabled, config.RequestsPerSecond), NewPublicBlockchainAPI(hmy, V2, config.RateLimiterEnabled, config.RequestsPerSecond), - NewPublicContractAPI(hmy, V1, config.RateLimiterEnabled, config.RequestsPerSecond), - NewPublicContractAPI(hmy, V2, config.RateLimiterEnabled, config.RequestsPerSecond), + NewPublicContractAPI(hmy, V1, config.RateLimiterEnabled, config.RequestsPerSecond, config.EvmCallTimeout), + NewPublicContractAPI(hmy, V2, config.RateLimiterEnabled, config.RequestsPerSecond, config.EvmCallTimeout), NewPublicTransactionAPI(hmy, V1), NewPublicTransactionAPI(hmy, V2), NewPublicPoolAPI(hmy, V1, config.RateLimiterEnabled, config.RequestsPerSecond), @@ -185,7 +190,7 @@ func getAPIs(hmy *hmy.Harmony, config nodeconfig.RPCServerConfig) []rpc.API { publicAPIs = append(publicAPIs, NewPublicHarmonyAPI(hmy, Eth), NewPublicBlockchainAPI(hmy, Eth, config.RateLimiterEnabled, config.RequestsPerSecond), - NewPublicContractAPI(hmy, Eth, config.RateLimiterEnabled, config.RequestsPerSecond), + NewPublicContractAPI(hmy, Eth, config.RateLimiterEnabled, config.RequestsPerSecond, config.EvmCallTimeout), NewPublicTransactionAPI(hmy, Eth), NewPublicPoolAPI(hmy, Eth, config.RateLimiterEnabled, config.RequestsPerSecond), eth.NewPublicEthService(hmy, "eth"), @@ -210,7 +215,7 @@ func getAPIs(hmy *hmy.Harmony, config nodeconfig.RPCServerConfig) []rpc.API { return publicAPIs } -func startHTTP(apis []rpc.API, rmf *rpc.RpcMethodFilter) (err error) { +func startHTTP(apis []rpc.API, rmf *rpc.RpcMethodFilter, httpTimeouts rpc.HTTPTimeouts) (err error) { httpListener, httpHandler, err = rpc.StartHTTPEndpoint( httpEndpoint, apis, HTTPModules, rmf, httpOrigins, httpVirtualHosts, httpTimeouts, ) @@ -227,7 +232,7 @@ func startHTTP(apis []rpc.API, rmf *rpc.RpcMethodFilter) (err error) { return nil } -func startAuthHTTP(apis []rpc.API, rmf *rpc.RpcMethodFilter) (err error) { +func startAuthHTTP(apis []rpc.API, rmf *rpc.RpcMethodFilter, httpTimeouts rpc.HTTPTimeouts) (err error) { httpListener, httpHandler, err = rpc.StartHTTPEndpoint( httpAuthEndpoint, apis, HTTPModules, rmf, httpOrigins, httpVirtualHosts, httpTimeouts, ) From 832d09d9b51b13717dfe224d14cc0b0c3565f6ed Mon Sep 17 00:00:00 2001 From: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Date: Thu, 23 Feb 2023 09:24:23 +0000 Subject: [PATCH 136/420] remove default timeouts --- rosetta/infra/harmony-mainnet.conf | 26 +++++++++++++++++++++----- rosetta/infra/harmony-pstn.conf | 26 +++++++++++++++++++++----- rosetta/services/construction_check.go | 2 +- rpc/rpc.go | 8 ++------ 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/rosetta/infra/harmony-mainnet.conf b/rosetta/infra/harmony-mainnet.conf index a929eeeab1..8d51609cb1 100644 --- a/rosetta/infra/harmony-mainnet.conf +++ b/rosetta/infra/harmony-mainnet.conf @@ -1,4 +1,4 @@ -Version = "2.5.11" +Version = "2.5.13" [BLSKeys] KMSConfigFile = "" @@ -41,9 +41,12 @@ Version = "2.5.11" AuthPort = 9501 Enabled = true IP = "0.0.0.0" + IdleTimeout = "120s" Port = 9500 + ReadTimeout = "30s" RosettaEnabled = true RosettaPort = 9700 + WriteTimeout = "30s" [Log] Console = false @@ -62,6 +65,8 @@ Version = "2.5.11" NetworkType = "mainnet" [P2P] + ConnManagerHighWatermark = 192 + ConnManagerLowWatermark = 160 DisablePrivateIPScan = false DiscConcurrency = 0 IP = "0.0.0.0" @@ -69,8 +74,6 @@ Version = "2.5.11" MaxConnsPerIP = 10 MaxPeers = 0 Port = 9000 - ConnManagerLowWatermark = 160 - ConnManagerHighWatermark = 192 WaitForEachPeerToConnect = false [Pprof] @@ -84,6 +87,7 @@ Version = "2.5.11" [RPCOpt] DebugEnabled = false EthRPCsEnabled = true + EvmCallTimeout = "5s" LegacyRPCsEnabled = true RateLimterEnabled = true RequestsPerSecond = 1000 @@ -104,11 +108,23 @@ Version = "2.5.11" DiscHighCap = 128 DiscSoftLowCap = 8 Downloader = false - StagedSync = false Enabled = false InitStreams = 8 - MinPeers = 5 MaxAdvertiseWaitTime = 30 + MinPeers = 5 + StagedSync = false + + [Sync.StagedSyncCfg] + DoubleCheckBlockHashes = false + InsertChainBatchSize = 0 + LogProgress = false + MaxBackgroundBlocks = 0 + MaxBlocksPerSyncCycle = 0 + MaxMemSyncCycleSize = 0 + TurboMode = false + UseMemDB = false + VerifyAllSig = false + VerifyHeaderBatchSize = 0 [TxPool] AccountSlots = 16 diff --git a/rosetta/infra/harmony-pstn.conf b/rosetta/infra/harmony-pstn.conf index edb911f87a..1bb865c1a5 100644 --- a/rosetta/infra/harmony-pstn.conf +++ b/rosetta/infra/harmony-pstn.conf @@ -1,4 +1,4 @@ -Version = "2.5.11" +Version = "2.5.13" [BLSKeys] KMSConfigFile = "" @@ -41,9 +41,12 @@ Version = "2.5.11" AuthPort = 9501 Enabled = true IP = "0.0.0.0" + IdleTimeout = "120s" Port = 9500 + ReadTimeout = "30s" RosettaEnabled = true RosettaPort = 9700 + WriteTimeout = "30s" [Log] Console = false @@ -62,6 +65,8 @@ Version = "2.5.11" NetworkType = "partner" [P2P] + ConnManagerHighWatermark = 192 + ConnManagerLowWatermark = 160 DisablePrivateIPScan = false DiscConcurrency = 0 IP = "0.0.0.0" @@ -69,8 +74,6 @@ Version = "2.5.11" MaxConnsPerIP = 10 MaxPeers = 0 Port = 9000 - ConnManagerLowWatermark = 160 - ConnManagerHighWatermark = 192 WaitForEachPeerToConnect = false [Pprof] @@ -84,6 +87,7 @@ Version = "2.5.11" [RPCOpt] DebugEnabled = false EthRPCsEnabled = true + EvmCallTimeout = "5s" LegacyRPCsEnabled = true RateLimterEnabled = true RequestsPerSecond = 1000 @@ -104,11 +108,23 @@ Version = "2.5.11" DiscHighCap = 128 DiscSoftLowCap = 8 Downloader = false - StagedSync = false Enabled = false InitStreams = 8 - MinPeers = 2 MaxAdvertiseWaitTime = 30 + MinPeers = 2 + StagedSync = false + + [Sync.StagedSyncCfg] + DoubleCheckBlockHashes = false + InsertChainBatchSize = 0 + LogProgress = false + MaxBackgroundBlocks = 0 + MaxBlocksPerSyncCycle = 0 + MaxMemSyncCycleSize = 0 + TurboMode = false + UseMemDB = false + VerifyAllSig = false + VerifyHeaderBatchSize = 0 [TxPool] AccountSlots = 16 diff --git a/rosetta/services/construction_check.go b/rosetta/services/construction_check.go index 1c3e1db699..f60296971c 100644 --- a/rosetta/services/construction_check.go +++ b/rosetta/services/construction_check.go @@ -270,7 +270,7 @@ func (s *ConstructAPI) ConstructionMetadata( callArgs.To = &contractAddress } evmExe, err := rpc.DoEVMCall( - ctx, s.hmy, callArgs, latest, rpc.CallTimeout, + ctx, s.hmy, callArgs, latest, s.hmy.NodeAPI.GetConfig().NodeConfig.RPCServer.EvmCallTimeout, ) if err != nil { return nil, common.NewError(common.CatchAllError, map[string]interface{}{ diff --git a/rpc/rpc.go b/rpc/rpc.go index 1bae7367de..a8f1e121a9 100644 --- a/rpc/rpc.go +++ b/rpc/rpc.go @@ -4,7 +4,6 @@ import ( "fmt" "net" "strings" - "time" "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy" @@ -28,8 +27,6 @@ const ( const ( // APIVersion used for DApp's, bumped after RPC refactor (7/2020) APIVersion = "1.1" - // CallTimeout is the timeout given to all contract calls - CallTimeout = 5 * time.Second // LogTag is the tag found in the log for all RPC logs LogTag = "[RPC]" // HTTPPortOffset .. @@ -58,9 +55,8 @@ var ( wsEndpoint = "" wsAuthEndpoint = "" httpVirtualHosts = []string{"*"} - // httpTimeouts = rpc.DefaultHTTPTimeouts - httpOrigins = []string{"*"} - wsOrigins = []string{"*"} + httpOrigins = []string{"*"} + wsOrigins = []string{"*"} ) // Version of the RPC From afdbd44bd50ccaa4d86a6ec90e7ab9a2f5b01139 Mon Sep 17 00:00:00 2001 From: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Date: Thu, 23 Feb 2023 09:29:06 +0000 Subject: [PATCH 137/420] store the evm call timeout in rosetta object --- rosetta/rosetta.go | 3 ++- rosetta/services/construction.go | 15 +++++++++------ rosetta/services/construction_check.go | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/rosetta/rosetta.go b/rosetta/rosetta.go index 5ff222a65b..a056300573 100644 --- a/rosetta/rosetta.go +++ b/rosetta/rosetta.go @@ -85,7 +85,8 @@ func getRouter(asserter *asserter.Asserter, hmy *hmy.Harmony, limiterEnable bool server.NewNetworkAPIController(services.NewNetworkAPI(hmy), asserter), server.NewConstructionAPIController(services.NewConstructionAPI(hmy), asserter), server.NewCallAPIController( - services.NewCallAPIService(hmy, limiterEnable, rateLimit, hmy.NodeAPI.GetConfig().NodeConfig.RPCServer.EvmCallTimeout), + services.NewCallAPIService(hmy, limiterEnable, rateLimit, + hmy.NodeAPI.GetConfig().NodeConfig.RPCServer.EvmCallTimeout), asserter, ), server.NewEventsAPIController(services.NewEventAPI(hmy), asserter), diff --git a/rosetta/services/construction.go b/rosetta/services/construction.go index 3442d4c629..f2bd6e6c91 100644 --- a/rosetta/services/construction.go +++ b/rosetta/services/construction.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/big" + "time" "github.com/coinbase/rosetta-sdk-go/server" "github.com/coinbase/rosetta-sdk-go/types" @@ -24,17 +25,19 @@ const ( // ConstructAPI implements the server.ConstructAPIServicer interface. type ConstructAPI struct { - hmy *hmy.Harmony - signer hmyTypes.Signer - stakingSigner stakingTypes.Signer + hmy *hmy.Harmony + signer hmyTypes.Signer + stakingSigner stakingTypes.Signer + evmCallTimeout time.Duration } // NewConstructionAPI creates a new instance of a ConstructAPI. func NewConstructionAPI(hmy *hmy.Harmony) server.ConstructionAPIServicer { return &ConstructAPI{ - hmy: hmy, - signer: hmyTypes.NewEIP155Signer(new(big.Int).SetUint64(hmy.ChainID)), - stakingSigner: stakingTypes.NewEIP155Signer(new(big.Int).SetUint64(hmy.ChainID)), + hmy: hmy, + signer: hmyTypes.NewEIP155Signer(new(big.Int).SetUint64(hmy.ChainID)), + stakingSigner: stakingTypes.NewEIP155Signer(new(big.Int).SetUint64(hmy.ChainID)), + evmCallTimeout: hmy.NodeAPI.GetConfig().NodeConfig.RPCServer.EvmCallTimeout, } } diff --git a/rosetta/services/construction_check.go b/rosetta/services/construction_check.go index f60296971c..c842770ab7 100644 --- a/rosetta/services/construction_check.go +++ b/rosetta/services/construction_check.go @@ -270,7 +270,7 @@ func (s *ConstructAPI) ConstructionMetadata( callArgs.To = &contractAddress } evmExe, err := rpc.DoEVMCall( - ctx, s.hmy, callArgs, latest, s.hmy.NodeAPI.GetConfig().NodeConfig.RPCServer.EvmCallTimeout, + ctx, s.hmy, callArgs, latest, s.evmCallTimeout, ) if err != nil { return nil, common.NewError(common.CatchAllError, map[string]interface{}{ From a96fb3f0d79a9ae61bb48ebfb5ec7df1973c179f Mon Sep 17 00:00:00 2001 From: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Date: Mon, 27 Feb 2023 04:11:01 +0000 Subject: [PATCH 138/420] [cmd] actually apply ToRPCServerConfig --- cmd/harmony/main.go | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index eacc392f10..f01cb758ed 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -334,23 +334,7 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { } // Parse RPC config - nodeConfig.RPCServer = nodeconfig.RPCServerConfig{ - HTTPEnabled: hc.HTTP.Enabled, - HTTPIp: hc.HTTP.IP, - HTTPPort: hc.HTTP.Port, - HTTPAuthPort: hc.HTTP.AuthPort, - WSEnabled: hc.WS.Enabled, - WSIp: hc.WS.IP, - WSPort: hc.WS.Port, - WSAuthPort: hc.WS.AuthPort, - DebugEnabled: hc.RPCOpt.DebugEnabled, - EthRPCsEnabled: hc.RPCOpt.EthRPCsEnabled, - StakingRPCsEnabled: hc.RPCOpt.StakingRPCsEnabled, - LegacyRPCsEnabled: hc.RPCOpt.LegacyRPCsEnabled, - RpcFilterFile: hc.RPCOpt.RpcFilterFile, - RateLimiterEnabled: hc.RPCOpt.RateLimterEnabled, - RequestsPerSecond: hc.RPCOpt.RequestsPerSecond, - } + nodeConfig.RPCServer = hc.ToRPCServerConfig() // Parse rosetta config nodeConfig.RosettaServer = nodeconfig.RosettaServerConfig{ From c47317339dcd4750b92ad6962162420876415d58 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 21 Feb 2023 15:20:47 -0300 Subject: [PATCH 139/420] Removed unused method. --- consensus/quorum/quorum.go | 1 - 1 file changed, 1 deletion(-) diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 6cf79e549a..da1551cdf9 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -98,7 +98,6 @@ type SignatoryTracker interface { // SignatureReader .. type SignatureReader interface { SignatoryTracker - ReadAllBallots(Phase) []*votepower.Ballot ReadBallot(p Phase, pubkey bls.SerializedPublicKey) *votepower.Ballot TwoThirdsSignersCount() int64 // 96 bytes aggregated signature From 51a0d70a6e790384f8750cc5a620d59a4bc078a2 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 21 Feb 2023 18:45:17 -0300 Subject: [PATCH 140/420] Rotate external leaders on non-beacon chains. --- consensus/consensus_v2.go | 10 +++++++++- consensus/quorum/quorum.go | 1 + consensus/view_change.go | 16 +++++++++++----- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 6b0d54648d..99bccf7554 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -714,7 +714,15 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { } } // Passed all checks, we can change leader. - wasFound, next := consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) + var ( + wasFound bool + next *bls.PublicKeyWrapper + ) + if consensus.ShardID == shard.BeaconChainShardID { + wasFound, next = consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) + } else { + wasFound, next = consensus.Decider.NthNext(leader, 1) + } if !wasFound { utils.Logger().Error().Msg("Failed to get next leader") return diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index da1551cdf9..aba62fa53f 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -75,6 +75,7 @@ type ParticipantTracker interface { Participants() multibls.PublicKeys IndexOf(bls.SerializedPublicKey) int ParticipantsCount() int64 + NthNext(*bls.PublicKeyWrapper, int) (bool, *bls.PublicKeyWrapper) NthNextHmy(shardingconfig.Instance, *bls.PublicKeyWrapper, int) (bool, *bls.PublicKeyWrapper) NthNextHmyExt(shardingconfig.Instance, *bls.PublicKeyWrapper, int) (bool, *bls.PublicKeyWrapper) FirstParticipant(shardingconfig.Instance) *bls.PublicKeyWrapper diff --git a/consensus/view_change.go b/consensus/view_change.go index d8b1058a04..899a50121d 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -202,11 +202,17 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe // FIXME: rotate leader on harmony nodes only before fully externalization var wasFound bool var next *bls.PublicKeyWrapper - if blockchain != nil && blockchain.Config().IsAllowlistEpoch(epoch) { - wasFound, next = consensus.Decider.NthNextHmyExt( - shard.Schedule.InstanceForEpoch(epoch), - lastLeaderPubKey, - gap) + if consensus.Blockchain().Config().IsLeaderRotation(epoch) { + if consensus.ShardID == shard.BeaconChainShardID { + wasFound, next = consensus.Decider.NthNextHmy( + shard.Schedule.InstanceForEpoch(epoch), + lastLeaderPubKey, + gap) + } else { + wasFound, next = consensus.Decider.NthNext( + lastLeaderPubKey, + gap) + } } else { wasFound, next = consensus.Decider.NthNextHmy( shard.Schedule.InstanceForEpoch(epoch), From 3d2916bb0824dac1b6eebc4ee46b892d08f87281 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 21 Feb 2023 22:33:57 -0300 Subject: [PATCH 141/420] Fix nil panic. --- consensus/view_change.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/view_change.go b/consensus/view_change.go index 899a50121d..aafdfd1210 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -202,7 +202,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe // FIXME: rotate leader on harmony nodes only before fully externalization var wasFound bool var next *bls.PublicKeyWrapper - if consensus.Blockchain().Config().IsLeaderRotation(epoch) { + if blockchain != nil && blockchain.Config().IsLeaderRotation(epoch) { if consensus.ShardID == shard.BeaconChainShardID { wasFound, next = consensus.Decider.NthNextHmy( shard.Schedule.InstanceForEpoch(epoch), From 8b051d3c64ab2f89c34430d4e878c026af697c41 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Feb 2023 16:27:17 +0000 Subject: [PATCH 142/420] Bump github.com/aws/aws-sdk-go from 1.33.0 to 1.34.0 Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.33.0 to 1.34.0. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/v1.34.0/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.33.0...v1.34.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 561a021ce8..d1d4f2bec7 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/VictoriaMetrics/fastcache v1.5.7 github.com/Workiva/go-datastructures v1.0.50 github.com/allegro/bigcache v1.2.1 - github.com/aws/aws-sdk-go v1.33.0 + github.com/aws/aws-sdk-go v1.34.0 github.com/beevik/ntp v0.3.0 github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce github.com/cespare/cp v1.1.1 diff --git a/go.sum b/go.sum index 48d1761c4a..7b2ab6d5d1 100644 --- a/go.sum +++ b/go.sum @@ -95,8 +95,8 @@ github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1: github.com/aristanetworks/goarista v0.0.0-20190607111240-52c2a7864a08 h1:UxoB3EYChE92EDNqRCS5vuE2ta4L/oKpeFaCK73KGvI= github.com/aristanetworks/goarista v0.0.0-20190607111240-52c2a7864a08/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/aws/aws-sdk-go v1.33.0 h1:Bq5Y6VTLbfnJp1IV8EL/qUU5qO1DYHda/zis/sqevkY= -github.com/aws/aws-sdk-go v1.33.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.34.0 h1:brux2dRrlwCF5JhTL7MUT3WUwo9zfDHZZp3+g3Mvlmo= +github.com/aws/aws-sdk-go v1.34.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From 3a82b3eeddacd46bde8fb384195f9803d57032a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Feb 2023 22:08:07 +0000 Subject: [PATCH 143/420] Bump github.com/ipld/go-ipld-prime from 0.9.0 to 0.19.0 Bumps [github.com/ipld/go-ipld-prime](https://github.com/ipld/go-ipld-prime) from 0.9.0 to 0.19.0. - [Release notes](https://github.com/ipld/go-ipld-prime/releases) - [Changelog](https://github.com/ipld/go-ipld-prime/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipld/go-ipld-prime/compare/v0.9.0...v0.19.0) --- updated-dependencies: - dependency-name: github.com/ipld/go-ipld-prime dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 20 ++++---------------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index d1d4f2bec7..9c9db1a29d 100644 --- a/go.mod +++ b/go.mod @@ -150,7 +150,7 @@ require ( github.com/ipfs/go-ipns v0.2.0 // indirect github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-log/v2 v2.5.1 // indirect - github.com/ipld/go-ipld-prime v0.9.0 // indirect + github.com/ipld/go-ipld-prime v0.19.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect @@ -215,7 +215,7 @@ require ( github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898 // indirect github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 // indirect + github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect diff --git a/go.sum b/go.sum index 7b2ab6d5d1..62fcf766b0 100644 --- a/go.sum +++ b/go.sum @@ -245,7 +245,6 @@ github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -493,7 +492,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= -github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= @@ -514,8 +512,8 @@ github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscw github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= -github.com/ipld/go-ipld-prime v0.9.0 h1:N2OjJMb+fhyFPwPnVvJcWU/NsumP8etal+d2v3G4eww= -github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= +github.com/ipld/go-ipld-prime v0.19.0 h1:5axC7rJmPc17Emw6TelxGwnzALk0PdupZ2oj2roDj04= +github.com/ipld/go-ipld-prime v0.19.0/go.mod h1:Q9j3BaVXwaA3o5JUDNvptDDr/x8+F7FG6XJ8WI3ILg4= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= @@ -693,14 +691,12 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= -github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= -github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= @@ -713,22 +709,18 @@ github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2 github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= -github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= github.com/multiformats/go-multicodec v0.7.0 h1:rTUjGOwjlhGHbEMbPoSUJowG1spZTVsITRANCjKTUAQ= github.com/multiformats/go-multicodec v0.7.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= -github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= github.com/multiformats/go-multistream v0.3.3 h1:d5PZpjwRgVlbwfdTDjife7XszfZd8KYWfROYFlGcR8o= github.com/multiformats/go-multistream v0.3.3/go.mod h1:ODRoqamLUsETKS9BNcII4gcRsJBU5VAwRIv7O39cEXg= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -808,8 +800,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 h1:CskT+S6Ay54OwxBGB0R3Rsx4Muto6UnEYTyKJbyRIAI= -github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e h1:ZOcivgkkFRnjfoTcGsDq3UQYiBmekwLA+qg0OjyB/ls= +github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -1014,7 +1006,6 @@ github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMI github.com/vmihailenco/msgpack/v5 v5.1.4/go.mod h1:C5gboKD0TJPqWDTVTtrQNfRbiBwHZGo8UTqP/9/XvLI= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= -github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= @@ -1102,7 +1093,6 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -1296,7 +1286,6 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1313,7 +1302,6 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= From d033f3612e54c4b6e6d43b3ebc6e690ff64d7fc3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Feb 2023 03:34:23 +0000 Subject: [PATCH 144/420] Bump golang.org/x/net from 0.3.0 to 0.7.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.3.0 to 0.7.0. - [Release notes](https://github.com/golang/net/releases) - [Commits](https://github.com/golang/net/compare/v0.3.0...v0.7.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 9c9db1a29d..18e16a0c05 100644 --- a/go.mod +++ b/go.mod @@ -54,9 +54,9 @@ require ( go.uber.org/ratelimit v0.1.0 go.uber.org/zap v1.24.0 golang.org/x/crypto v0.4.0 - golang.org/x/net v0.3.0 // indirect + golang.org/x/net v0.7.0 // indirect golang.org/x/sync v0.1.0 - golang.org/x/sys v0.3.0 // indirect + golang.org/x/sys v0.5.0 // indirect golang.org/x/time v0.2.0 golang.org/x/tools v0.3.0 // indirect google.golang.org/grpc v1.51.0 @@ -254,8 +254,8 @@ require ( go.uber.org/multierr v1.8.0 // indirect golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect golang.org/x/mod v0.7.0 // indirect - golang.org/x/term v0.3.0 // indirect - golang.org/x/text v0.5.0 // indirect + golang.org/x/term v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 62fcf766b0..2ebb5e94a2 100644 --- a/go.sum +++ b/go.sum @@ -1193,8 +1193,8 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.3.0 h1:VWL6FNY2bEEmsGVKabSlHu5Irp34xmMRoqb/9lF9lxk= -golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1300,12 +1300,12 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1315,8 +1315,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From a240abb11903c59067bfdbca08f6ca2141388566 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 16 Aug 2022 02:04:03 +0300 Subject: [PATCH 145/420] Small fixes. --- cmd/harmony/main.go | 9 ++++----- consensus/view_change_construct.go | 2 +- rpc/blockchain.go | 3 +-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index da51d9c52e..437a91dc59 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -769,7 +769,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi ) nodeconfig.GetDefaultConfig().DBDir = nodeConfig.DBDir - processNodeType(hc, currentNode, currentConsensus) + currentConsensus.SetIsBackup(processNodeType(hc, currentNode)) currentNode.NodeConfig.SetShardGroupID(nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(nodeConfig.ShardID))) currentNode.NodeConfig.SetClientGroupID(nodeconfig.NewClientGroupIDByShardID(shard.BeaconChainShardID)) currentNode.NodeConfig.ConsensusPriKey = nodeConfig.ConsensusPriKey @@ -821,7 +821,7 @@ func setupTiKV(hc harmonyconfig.HarmonyConfig) shardchain.DBFactory { return factory } -func processNodeType(hc harmonyconfig.HarmonyConfig, currentNode *node.Node, currentConsensus *consensus.Consensus) { +func processNodeType(hc harmonyconfig.HarmonyConfig, currentNode *node.Node) (isBackup bool) { switch hc.General.NodeType { case nodeTypeExplorer: nodeconfig.SetDefaultRole(nodeconfig.ExplorerNode) @@ -831,10 +831,9 @@ func processNodeType(hc harmonyconfig.HarmonyConfig, currentNode *node.Node, cur nodeconfig.SetDefaultRole(nodeconfig.Validator) currentNode.NodeConfig.SetRole(nodeconfig.Validator) - if hc.General.IsBackup { - currentConsensus.SetIsBackup(true) - } + return hc.General.IsBackup } + return false } func setupPprofService(node *node.Node, hc harmonyconfig.HarmonyConfig) { diff --git a/consensus/view_change_construct.go b/consensus/view_change_construct.go index b818507eb7..52ea8bbd72 100644 --- a/consensus/view_change_construct.go +++ b/consensus/view_change_construct.go @@ -87,7 +87,7 @@ func (vc *viewChange) AddViewIDKeyIfNotExist(viewID uint64, members multibls.Pub } } -// Reset reset the state for viewchange +// Reset resets the state for viewChange. func (vc *viewChange) Reset() { vc.m1Payload = []byte{} vc.bhpSigs = map[uint64]map[string]*bls_core.Sign{} diff --git a/rpc/blockchain.go b/rpc/blockchain.go index 6f3e2293f7..46e495944d 100644 --- a/rpc/blockchain.go +++ b/rpc/blockchain.go @@ -2,13 +2,12 @@ package rpc import ( "context" + "encoding/hex" "fmt" "math/big" "reflect" "time" - "encoding/hex" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" From 6bc405c81816028cda25f68945eb5dabf525a96a Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 1 Sep 2022 00:53:16 +0700 Subject: [PATCH 146/420] in progress. --- api/service/blockproposal/service.go | 15 +++++---- api/service/explorer/service.go | 48 ++++++++++++++++++++++++++++ cmd/harmony/main.go | 6 +++- consensus/checks.go | 3 +- consensus/consensus.go | 6 +++- consensus/consensus_service.go | 48 ++++++++++++++++++++++++---- consensus/consensus_v2.go | 33 +++++++++++++++++-- consensus/leader.go | 12 +++++-- consensus/quorum/quorum.go | 7 +++- consensus/validator.go | 14 ++++++++ consensus/view_change.go | 30 +++++++++++++++-- hmy/hmy.go | 2 ++ internal/utils/singleton.go | 16 ++++++++++ node/api.go | 6 ++++ node/node_newblock.go | 4 +++ test/configs/local-resharding.txt | 31 ++++++------------ test/deploy.sh | 5 ++- 17 files changed, 237 insertions(+), 49 deletions(-) diff --git a/api/service/blockproposal/service.go b/api/service/blockproposal/service.go index 4d42c558aa..1cbb5accf3 100644 --- a/api/service/blockproposal/service.go +++ b/api/service/blockproposal/service.go @@ -18,19 +18,22 @@ type Service struct { // New returns a block proposal service. func New(readySignal chan consensus.ProposalType, commitSigsChan chan []byte, waitForConsensusReady func(readySignal chan consensus.ProposalType, commitSigsChan chan []byte, stopChan chan struct{}, stoppedChan chan struct{})) *Service { - return &Service{readySignal: readySignal, commitSigsChan: commitSigsChan, waitForConsensusReady: waitForConsensusReady} + return &Service{ + readySignal: readySignal, + commitSigsChan: commitSigsChan, + waitForConsensusReady: waitForConsensusReady, + stopChan: make(chan struct{}), + stoppedChan: make(chan struct{}), + } } // Start starts block proposal service. func (s *Service) Start() error { - s.stopChan = make(chan struct{}) - s.stoppedChan = make(chan struct{}) - - s.run(s.stopChan, s.stoppedChan) + s.run() return nil } -func (s *Service) run(stopChan chan struct{}, stoppedChan chan struct{}) { +func (s *Service) run() { s.waitForConsensusReady(s.readySignal, s.commitSigsChan, s.stopChan, s.stoppedChan) } diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index b5ffa1b8b0..d96c1bc56c 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -7,11 +7,13 @@ import ( "fmt" "net" "net/http" + "os" "path" "strconv" "time" "github.com/RoaringBitmap/roaring/roaring64" + "github.com/harmony-one/harmony/internal/common" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" ethCommon "github.com/ethereum/go-ethereum/common" @@ -113,12 +115,17 @@ func (s *Service) Run() *http.Server { s.router = mux.NewRouter() + fmt.Println("++", addr) + // Set up router for addresses. // Fetch addresses request, accepts parameter size: how much addresses to read, // parameter prefix: from which address prefix start s.router.Path("/addresses").Queries("size", "{[0-9]*?}", "prefix", "{[a-zA-Z0-9]*?}").HandlerFunc(s.GetAddresses).Methods("GET") s.router.Path("/addresses").HandlerFunc(s.GetAddresses) s.router.Path("/height").HandlerFunc(s.GetHeight) + s.router.Path("/leader").HandlerFunc(s.GetLeader) + s.router.Path("/blocks").HandlerFunc(s.GetBlocks) + s.router.Path("/halt").HandlerFunc(s.halt) // Set up router for supply info s.router.Path("/burn-addresses").Queries().HandlerFunc(s.GetInaccessibleAddressInfo).Methods("GET") @@ -186,6 +193,47 @@ type HeightResponse struct { S3 uint64 `json:"3,omitempty"` } +func (s *Service) GetLeader(w http.ResponseWriter, r *http.Request) { + if s.backend.IsCurrentlyLeader() { + w.Write([]byte("true ")) + } else { + w.Write([]byte("false")) + } + + keys := "" + for _, p := range s.backend.GetPublicKeys() { + addr := common.Address{} + addrBytes := p.Object.GetAddress() + addr.SetBytes(addrBytes[:]) + keys += fmt.Sprintf("%s ", addr.String()) + break + } + //blsPubKeyBytes := leaderKey.Object.GetAddress() + //coinbase.SetBytes(blsPubKeyBytes[:]) + + w.Write([]byte(fmt.Sprintf(" %d", s.blockchain.ShardID()))) + w.Write([]byte(fmt.Sprintf(" %s", s.Port))) + w.Write([]byte(fmt.Sprintf(" %s", keys))) + w.Write([]byte(fmt.Sprintf(" %s", s.backend.GetPublicKeys().SerializeToHexStr()))) + +} + +func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { + cur := s.blockchain.CurrentHeader().Number().Uint64() + + for i := cur; i > 0; i-- { + block := s.blockchain.GetBlockByNumber(i) + w.Write([]byte(fmt.Sprintf("%d ", i))) + w.Write([]byte(fmt.Sprintf("%s ", block.Header().ViewID().String()))) + w.Write([]byte(fmt.Sprintf("%s ", block.Header().Coinbase().Hash().Hex()))) + w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) + } +} + +func (s *Service) halt(w http.ResponseWriter, r *http.Request) { + os.Exit(0) +} + // GetHeight returns heights of current and beacon chains if needed. func (s *Service) GetHeight(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 437a91dc59..75eabfb2e8 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -265,6 +265,8 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { var err error + fmt.Println("OS: ", os.Args) + nodeconfigSetShardSchedule(hc) nodeconfig.SetShardingSchedule(shard.Schedule) nodeconfig.SetVersion(getHarmonyVersion()) @@ -420,8 +422,9 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { if currentNode.NodeConfig.Role() == nodeconfig.Validator { currentNode.RegisterValidatorServices() } else if currentNode.NodeConfig.Role() == nodeconfig.ExplorerNode { - currentNode.RegisterExplorerServices() + } + currentNode.RegisterExplorerServices() currentNode.RegisterService(service.CrosslinkSending, crosslink_sending.New(currentNode, currentNode.Blockchain())) if hc.Pprof.Enabled { setupPprofService(currentNode, hc) @@ -784,6 +787,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // Set the consensus ID to be the current block number viewID := currentNode.Blockchain().CurrentBlock().Header().ViewID().Uint64() + fmt.Println("viewID:", viewID) currentConsensus.SetViewIDs(viewID + 1) utils.Logger().Info(). Uint64("viewID", viewID). diff --git a/consensus/checks.go b/consensus/checks.go index ceaf9987b9..c44a58da39 100644 --- a/consensus/checks.go +++ b/consensus/checks.go @@ -55,8 +55,7 @@ func (consensus *Consensus) senderKeySanityChecks(msg *msg_pb.Message, senderKey return true } -func (consensus *Consensus) isRightBlockNumAndViewID(recvMsg *FBFTMessage, -) bool { +func (consensus *Consensus) isRightBlockNumAndViewID(recvMsg *FBFTMessage) bool { if recvMsg.ViewID != consensus.GetCurBlockViewID() || recvMsg.BlockNum != consensus.BlockNum() { consensus.getLogger().Debug(). Uint64("blockNum", consensus.BlockNum()). diff --git a/consensus/consensus.go b/consensus/consensus.go index 89897e372d..2598010102 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -73,6 +73,8 @@ type Consensus struct { priKey multibls.PrivateKeys // the publickey of leader LeaderPubKey *bls.PublicKeyWrapper + // index of leader in the list of validators. + LeaderIndex int // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 @@ -220,7 +222,9 @@ func New( registry *registry.Registry, Decider quorum.Decider, minPeers int, aggregateSig bool, ) (*Consensus, error) { - consensus := Consensus{} + consensus := Consensus{ + ShardID: shard, + } consensus.Decider = Decider consensus.registry = registry consensus.MinPeers = minPeers diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 310c9bb9da..e646504dde 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync/atomic" "time" @@ -106,11 +107,6 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi return consensus.Decider.ParticipantsCount() } -// NewFaker returns a faker consensus. -func NewFaker() *Consensus { - return &Consensus{} -} - // Sign on the hash of the message func (consensus *Consensus) signMessage(message []byte, priKey *bls_core.SecretKey) []byte { hash := hash.Keccak256(message) @@ -217,6 +213,7 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } + fmt.Println("[checkViewID] Set LEADEER PUB KEY ", msg.SenderPubkeys[0].Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() @@ -400,6 +397,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { Str("leaderPubKey", leaderPubKey.Bytes.Hex()). Msg("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain") consensus.pubKeyLock.Lock() + fmt.Println("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain", leaderPubKey.Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = leaderPubKey consensus.pubKeyLock.Unlock() } @@ -451,16 +449,43 @@ func (consensus *Consensus) IsLeader() bool { return false } +// isLeader check if the node is a leader or not by comparing the public key of +// the node with the leader public key. This function assume it runs under lock. +func (consensus *Consensus) isLeader() bool { + obj := consensus.LeaderPubKey.Object + for _, key := range consensus.priKey { + if key.Pub.Object.IsEqual(obj) { + return true + } + } + return false +} + // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { + fmt.Println("SetViewIDs", height) consensus.SetCurBlockViewID(height) consensus.SetViewChangingID(height) } // SetCurBlockViewID set the current view ID -func (consensus *Consensus) SetCurBlockViewID(viewID uint64) { - consensus.current.SetCurBlockViewID(viewID) +func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { + return consensus.current.SetCurBlockViewID(viewID) +} + +// SetLeaderIndex set the leader index. +func (consensus *Consensus) SetLeaderIndex(f func(int) int) (current int) { + consensus.pubKeyLock.Lock() + defer consensus.pubKeyLock.Unlock() + consensus.LeaderIndex = f(consensus.LeaderIndex) + return consensus.LeaderIndex +} + +func (consensus *Consensus) GetLeaderIndex() int { + consensus.pubKeyLock.Lock() + defer consensus.pubKeyLock.Unlock() + return consensus.LeaderIndex } // SetViewChangingID set the current view change ID @@ -473,6 +498,15 @@ func (consensus *Consensus) StartFinalityCount() { consensus.finalityCounter.Store(time.Now().UnixNano()) } +//func (consensus *Consensus) ReshardingNextLeader(newblock *types.Block) { +// consensus.pubKeyLock.Lock() +// fmt.Println("nextBlock1 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) +// consensus.LeaderPubKey = consensus.getNextLeaderKey(consensus.GetCurBlockViewID() + 1) +// fmt.Println("nextBlock2 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) +// consensus.pubKeyLock.Unlock() +// +//} + // FinishFinalityCount calculate the current finality func (consensus *Consensus) FinishFinalityCount() { d := time.Now().UnixNano() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index deb0883d94..16554d0393 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/hex" + "fmt" "sync/atomic" "time" @@ -130,6 +131,7 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb } func (consensus *Consensus) finalCommit() { + // THIS IS NOT GOOD PLACE FOR LEADER SWITCHING numCommits := consensus.Decider.SignersCount(quorum.Commit) consensus.getLogger().Info(). @@ -392,6 +394,7 @@ func (consensus *Consensus) Start( consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() case newBlock := <-blockChannel: + //consensus.ReshardingNextLeader(newBlock) consensus.getLogger().Info(). Uint64("MsgBlockNum", newBlock.NumberU64()). Msg("[ConsensusMainLoop] Received Proposed New Block!") @@ -403,6 +406,7 @@ func (consensus *Consensus) Start( } // Sleep to wait for the full block time consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") + <-time.After(time.Until(consensus.NextBlockDue)) consensus.StartFinalityCount() @@ -431,7 +435,7 @@ func (consensus *Consensus) Start( } } -// Close close the consensus. If current is in normal commit phase, wait until the commit +// Close closes the consensus. If current is in normal commit phase, wait until the commit // phase end. func (consensus *Consensus) Close() error { if consensus.dHelper != nil { @@ -527,6 +531,7 @@ func (consensus *Consensus) getLastMileBlocksAndMsg(bnStart uint64) ([]*types.Bl // preCommitAndPropose commit the current block with 67% commit signatures and start // proposing new block which will wait on the full commit signatures to finish func (consensus *Consensus) preCommitAndPropose(blk *types.Block) error { + //fmt.Println("preCommitAndPropose", utils.GetPort(), blk.NumberU64()) if blk == nil { return errors.New("block to pre-commit is nil") } @@ -644,6 +649,7 @@ func (consensus *Consensus) tryCatchup() error { consensus.getLogger().Error().Err(err).Msg("[TryCatchup] Failed to add block to chain") return err } + //fmt.Println("tryCatchup ", utils.GetPort(), blk.NumberU64()) select { // TODO: Remove this when removing dns sync and stream sync is fully up case consensus.VerifiedNewBlock <- blk: @@ -658,6 +664,8 @@ func (consensus *Consensus) tryCatchup() error { } func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMessage) error { + // this function evaluates for all, leader and validators. + if consensus.Blockchain().CurrentBlock().NumberU64() < blk.NumberU64() { if _, err := consensus.Blockchain().InsertChain([]*types.Block{blk}, !consensus.FBFTLog.IsBlockVerified(blk.Hash())); err != nil { consensus.getLogger().Error().Err(err).Msg("[commitBlock] Failed to add block to chain") @@ -682,10 +690,29 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess // SetupForNewConsensus sets the state for new consensus func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) - consensus.SetCurBlockViewID(committedMsg.ViewID + 1) + curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) // first view id is going to be 2. + prev := consensus.GetLeaderPubKey() + idx := consensus.SetLeaderIndex(func(i int) int { + if curBlockViewID%3 == 0 { + return i + 1 + } + return i + }) + pps := consensus.Decider.Participants() consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] + consensus.LeaderPubKey = &pps[idx%len(pps)] + fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %q\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) consensus.pubKeyLock.Unlock() + if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { + // leader changed + go func() { + fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + consensus.ReadySignal <- SyncProposal + + }() + + } // Update consensus keys at last so the change of leader status doesn't mess up normal flow if blk.IsLastBlockInEpoch() { consensus.SetMode(consensus.UpdateConsensusInformation()) diff --git a/consensus/leader.go b/consensus/leader.go index 477d8eb299..a359c229a4 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -3,12 +3,11 @@ package consensus import ( "time" + "github.com/harmony-one/harmony/consensus/signature" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/common" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" - "github.com/harmony-one/harmony/consensus/signature" - "github.com/ethereum/go-ethereum/rlp" bls_core "github.com/harmony-one/bls/ffi/go/bls" msg_pb "github.com/harmony-one/harmony/api/proto/message" @@ -200,9 +199,17 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { + // TODO HERE + //if recvMsg.ViewID == 10 { + // return + //} consensus.mutex.Lock() defer consensus.mutex.Unlock() //// Read - Start + if consensus.ShardID == 0 { + //fmt.Println("onCommit ", recvMsg.BlockNum) + } + if !consensus.isRightBlockNumAndViewID(recvMsg) { return } @@ -334,4 +341,5 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED) } + //fmt.Println("onCommit99: ", utils.GetPort(), recvMsg.BlockNum) } diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 867fd99677..7e76dfcc6b 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -231,12 +231,17 @@ func (s *cIdentities) NthNextHmy(instance shardingconfig.Instance, pubKey *bls.P Msg("[NthNextHmy] pubKey not found") } numNodes := instance.NumHarmonyOperatedNodesPerShard() + //fmt.Println("??idx:", idx, numNodes) // sanity check to avoid out of bound access if numNodes <= 0 || numNodes > len(s.publicKeys) { numNodes = len(s.publicKeys) } idx = (idx + next) % numNodes - return found, &s.publicKeys[idx] + //fmt.Println("-------idx:", idx) + + new := &s.publicKeys[idx] + fmt.Println("NthNextHmy: ", pubKey.Bytes.Hex(), new.Bytes.Hex()) + return found, new } // NthNextHmyExt return the Nth next pubkey of Harmony + allowlist nodes, next can be negative number diff --git a/consensus/validator.go b/consensus/validator.go index a73ac92eb3..013769395c 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -18,6 +18,7 @@ import ( ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { + recvMsg, err := consensus.ParseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). @@ -26,6 +27,9 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Msg("[OnAnnounce] Unparseable leader message") return } + if consensus.ShardID == 0 { + //fmt.Println("onAnnounce called ", recvMsg.BlockNum) + } // NOTE let it handle its own logs if !consensus.onAnnounceSanityChecks(recvMsg) { @@ -184,6 +188,9 @@ func (consensus *Consensus) sendCommitMessages(blockObj *types.Block) { // if onPrepared accepts the prepared message from the leader, then // it will send a COMMIT message for the leader to receive on the network. func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { + if consensus.ShardID == 0 { + //fmt.Println("onPrepared", recvMsg.BlockNum) + } consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -399,6 +406,13 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { consensus.getLogger().Info().Msg("[OnCommitted] Start consensus timer (new block added)") consensus.consensusTimeout[timeoutConsensus].Start() } + + //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum) + if blk != nil { + //consensus.ReshardingNextLeader(blk) + } else { + //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum, "blk is nil") + } } // Collect private keys that are part of the current committee. diff --git a/consensus/view_change.go b/consensus/view_change.go index 5bfd49f83d..2936fba018 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync" "time" @@ -67,10 +68,11 @@ func (pm *State) GetCurBlockViewID() uint64 { } // SetCurBlockViewID sets the current view id -func (pm *State) SetCurBlockViewID(viewID uint64) { +func (pm *State) SetCurBlockViewID(viewID uint64) uint64 { pm.cViewMux.Lock() defer pm.cViewMux.Unlock() pm.blockViewID = viewID + return pm.blockViewID } // GetViewChangingID return the current view changing id @@ -160,6 +162,7 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { Uint64("stuckBlockViewID", stuckBlockViewID). Msg("[getNextViewID]") + fmt.Println("end getNextViewID: ", nextViewID, viewChangeDuration) // duration is always the fixed view change duration for synchronous view change return nextViewID, viewChangeDuration } @@ -171,7 +174,8 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrapper { gap := 1 - if viewID > consensus.GetCurBlockViewID() { + cur := consensus.GetCurBlockViewID() + if viewID > cur { gap = int(viewID - consensus.GetCurBlockViewID()) } var lastLeaderPubKey *bls.PublicKeyWrapper @@ -231,6 +235,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe lastLeaderPubKey, gap) } + fmt.Println("wasfoundNext", consensus.Blockchain.Config().IsAllowlistEpoch(epoch), wasFound, next.Bytes.Hex(), lastLeaderPubKey.Bytes.Hex()) if !wasFound { consensus.getLogger().Warn(). Str("key", consensus.LeaderPubKey.Bytes.Hex()). @@ -252,6 +257,7 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { + fmt.Printf("Message to send leader111: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } @@ -262,6 +268,7 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() + //fmt.Println("startViewChange", nextViewID) consensus.SetViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet @@ -270,7 +277,10 @@ func (consensus *Consensus) startViewChange() { // Ideally, we shall use another variable to keep track of the // leader pubkey in viewchange mode consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = consensus.getNextLeaderKey(nextViewID) + lpk := consensus.getNextLeaderKey(nextViewID) + consensus.LeaderPubKey = lpk + //fmt.Println("Message to send leader cur: ", consensus.LeaderPubKey.Bytes.Hex(), "next: ", lpk.Bytes.Hex()) + //fmt.Println("Message to send leader: ", consensus.LeaderPubKey.Bytes.Hex()) consensus.pubKeyLock.Unlock() consensus.getLogger().Warn(). @@ -304,7 +314,9 @@ func (consensus *Consensus) startViewChange() { if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { continue } + // Тут уже другой leader msgToSend := consensus.constructViewChangeMessage(&key) + fmt.Println("Message to send leader222: ", consensus.LeaderPubKey.Bytes.Hex()) if err := consensus.msgSender.SendWithRetry( consensus.BlockNum(), msg_pb.MessageType_VIEWCHANGE, @@ -362,6 +374,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.ResetState() } + fmt.Println("[startNewView]", newLeaderPriKey.Pub.Bytes.Hex()) consensus.LeaderPubKey = newLeaderPriKey.Pub return nil @@ -369,6 +382,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri // onViewChange is called when the view change message is received. func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { + //fmt.Printf("[onViewChange] received view change message from %+v\n", recvMsg) consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -391,6 +405,13 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { return } + consensus.getLogger().Debug(). + Err(err). + Interface("SenderPubkeys", recvMsg.SenderPubkeys). + Str("NextLeader", recvMsg.LeaderPubkey.Bytes.Hex()). + Str("myBLSPubKey", consensus.priKey.GetPublicKeys().SerializeToHexStr()). + Msg("[onViewChange] I am the Leader") + if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) { consensus.getLogger().Info(). Int64("have", consensus.Decider.SignersCount(quorum.ViewChange)). @@ -471,6 +492,8 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.mutex.Lock() defer consensus.mutex.Unlock() + fmt.Printf("[onNewView] received new view message from %+v\n", recvMsg) + consensus.getLogger().Info(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -559,6 +582,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // newView message verified success, override my state consensus.SetViewIDs(recvMsg.ViewID) consensus.pubKeyLock.Lock() + fmt.Println("[onNewView1221] new leader key cur:", consensus.LeaderPubKey.Bytes.Hex(), " new: ", senderKey.Bytes.Hex()) consensus.LeaderPubKey = senderKey consensus.pubKeyLock.Unlock() consensus.ResetViewChangeState() diff --git a/hmy/hmy.go b/hmy/hmy.go index 58e8d59edf..8089f04ba8 100644 --- a/hmy/hmy.go +++ b/hmy/hmy.go @@ -18,6 +18,7 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + "github.com/harmony-one/harmony/multibls" commonRPC "github.com/harmony-one/harmony/rpc/common" "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" @@ -96,6 +97,7 @@ type NodeAPI interface { GetStakingTransactionsCount(address, txType string) (uint64, error) GetTraceResultByHash(hash common.Hash) (json.RawMessage, error) IsCurrentlyLeader() bool + GetPublicKeys() multibls.PublicKeys IsOutOfSync(shardID uint32) bool SyncStatus(shardID uint32) (bool, uint64, uint64) SyncPeers() map[string]int diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index 6409ea71e2..7aee4d86be 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path" + "strconv" "sync" "time" @@ -195,3 +196,18 @@ func updateZeroLogLevel(level int) { childLogger := Logger().Level(zeroLoggerLevel) zeroLogger = &childLogger } + +func GetPort() int { + ok := false + for _, x := range os.Args { + if x == "--port" { + ok = true + continue + } + if ok { + rs, _ := strconv.ParseInt(x, 10, 64) + return int(rs) + } + } + return 0 +} diff --git a/node/api.go b/node/api.go index ceda968084..debcb201f6 100644 --- a/node/api.go +++ b/node/api.go @@ -6,6 +6,7 @@ import ( "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy" "github.com/harmony-one/harmony/internal/tikv" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/rosetta" hmy_rpc "github.com/harmony-one/harmony/rpc" rpc_common "github.com/harmony-one/harmony/rpc/common" @@ -18,6 +19,11 @@ func (node *Node) IsCurrentlyLeader() bool { return node.Consensus.IsLeader() } +// GetPublicKeys exposes if node is currently the leader node +func (node *Node) GetPublicKeys() multibls.PublicKeys { + return node.Consensus.GetPrivateKeys().GetPublicKeys() +} + // PeerConnectivity .. func (node *Node) PeerConnectivity() (int, int, int) { return node.host.PeerConnectivity() diff --git a/node/node_newblock.go b/node/node_newblock.go index 03fd69d9dd..24e26ad99a 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -2,6 +2,7 @@ package node import ( "errors" + "fmt" "sort" "strings" "time" @@ -88,6 +89,8 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp newBlock, err := node.ProposeNewBlock(newCommitSigsChan) if err == nil { + fmt.Printf("ProposeNewBlock: #%d :%d @%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), newBlock.Header().ViewID().Int64(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) + if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). Msg("Block with the same number was already proposed, abort.") @@ -145,6 +148,7 @@ func (node *Node) ProposeNewBlock(commitSigs chan []byte) (*types.Block, error) if node.Blockchain().Config().IsStaking(header.Epoch()) { blsPubKeyBytes := leaderKey.Object.GetAddress() coinbase.SetBytes(blsPubKeyBytes[:]) + fmt.Println("coinbase.SetBytes leader: ", leaderKey.Bytes.Hex(), coinbase.Hex()) } emptyAddr := common.Address{} diff --git a/test/configs/local-resharding.txt b/test/configs/local-resharding.txt index 4ce91ece1c..fc3d3e97b4 100644 --- a/test/configs/local-resharding.txt +++ b/test/configs/local-resharding.txt @@ -1,25 +1,12 @@ 127.0.0.1 9000 validator .hmy/65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204.key -127.0.0.1 9002 validator .hmy/40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809.key -127.0.0.1 9004 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key -127.0.0.1 9006 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key -127.0.0.1 9008 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key -127.0.0.1 9010 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key -127.0.0.1 9012 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key -127.0.0.1 9014 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key -127.0.0.1 9016 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key -127.0.0.1 9018 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key -127.0.0.1 9020 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key -127.0.0.1 9022 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key -127.0.0.1 9200 explorer null 0 +127.0.0.1 9002 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key +127.0.0.1 9004 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key +127.0.0.1 9006 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key +127.0.0.1 9008 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key +127.0.0.1 9010 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key +127.0.0.1 9099 explorer null 0 127.0.0.1 9100 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key -127.0.0.1 9102 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key -127.0.0.1 9104 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key -127.0.0.1 9106 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key -127.0.0.1 9108 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key -127.0.0.1 9110 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key -127.0.0.1 9112 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key -127.0.0.1 9114 validator .hmy/f47238daef97d60deedbde5302d05dea5de67608f11f406576e363661f7dcbc4a1385948549b31a6c70f6fde8a391486.key -127.0.0.1 9116 validator .hmy/fc4b9c535ee91f015efff3f32fbb9d32cdd9bfc8a837bb3eee89b8fff653c7af2050a4e147ebe5c7233dc2d5df06ee0a.key -127.0.0.1 9118 validator .hmy/ca86e551ee42adaaa6477322d7db869d3e203c00d7b86c82ebee629ad79cb6d57b8f3db28336778ec2180e56a8e07296.key -127.0.0.1 9300 explorer null 1 \ No newline at end of file +127.0.0.1 9102 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key +127.0.0.1 9104 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key +127.0.0.1 9106 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key \ No newline at end of file diff --git a/test/deploy.sh b/test/deploy.sh index 6bbb12eb9b..5a949637e9 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -69,7 +69,7 @@ function launch_localnet() { if ${VERBOSE}; then verbosity=5 else - verbosity=3 + verbosity=5 fi base_args=(--log_folder "${log_folder}" --min_peers "${MIN}" --bootnodes "${BN_MA}" "--network_type=$NETWORK" --blspass file:"${ROOT}/.hmy/blspass.txt" "--dns=false" "--verbosity=${verbosity}" "--p2p.security.max-conn-per-ip=100") @@ -80,8 +80,11 @@ function launch_localnet() { while IFS='' read -r line || [[ -n "$line" ]]; do i=$((i + 1)) + + # Read config for i-th node form config file IFS=' ' read -r ip port mode bls_key shard node_config <<<"${line}" + echo "LINE: ${line} ${shard}" args=("${base_args[@]}" --ip "${ip}" --port "${port}" --key "/tmp/${ip}-${port}.key" --db_dir "${ROOT}/db-${ip}-${port}" "--broadcast_invalid_tx=false") if [[ -z "$ip" || -z "$port" ]]; then echo "skip empty node" From 9bd408033a88d225567d96e0c323acd0b16374e4 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 1 Sep 2022 01:00:51 +0700 Subject: [PATCH 147/420] in progress. --- consensus/consensus_v2.go | 1 - 1 file changed, 1 deletion(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 16554d0393..16fa18d565 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -131,7 +131,6 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb } func (consensus *Consensus) finalCommit() { - // THIS IS NOT GOOD PLACE FOR LEADER SWITCHING numCommits := consensus.Decider.SignersCount(quorum.Commit) consensus.getLogger().Info(). From bcbf5520ce31f627b329111a9465db0e02c3db9f Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 1 Sep 2022 01:02:36 +0700 Subject: [PATCH 148/420] in progress. --- consensus/consensus_v2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 16fa18d565..9ab924c004 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -689,7 +689,7 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess // SetupForNewConsensus sets the state for new consensus func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) - curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) // first view id is going to be 2. + curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) prev := consensus.GetLeaderPubKey() idx := consensus.SetLeaderIndex(func(i int) int { if curBlockViewID%3 == 0 { From f67d90be9d8bc7d57edfca25a8a19ec0528d63e3 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 5 Sep 2022 19:39:49 +0700 Subject: [PATCH 149/420] consensus check is forked --- api/service/explorer/service.go | 6 ++-- consensus/consensus.go | 2 -- consensus/consensus_service.go | 14 -------- consensus/consensus_v2.go | 63 +++++++++++++++++++++++---------- internal/params/config.go | 10 ++++++ 5 files changed, 58 insertions(+), 37 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index d96c1bc56c..5887bb3a02 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -223,9 +223,9 @@ func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { for i := cur; i > 0; i-- { block := s.blockchain.GetBlockByNumber(i) - w.Write([]byte(fmt.Sprintf("%d ", i))) - w.Write([]byte(fmt.Sprintf("%s ", block.Header().ViewID().String()))) - w.Write([]byte(fmt.Sprintf("%s ", block.Header().Coinbase().Hash().Hex()))) + w.Write([]byte(fmt.Sprintf("#%d ", i))) + w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) + w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) } } diff --git a/consensus/consensus.go b/consensus/consensus.go index 2598010102..fb7bf79b53 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -73,8 +73,6 @@ type Consensus struct { priKey multibls.PrivateKeys // the publickey of leader LeaderPubKey *bls.PublicKeyWrapper - // index of leader in the list of validators. - LeaderIndex int // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e646504dde..d26ea8ced5 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -474,20 +474,6 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } -// SetLeaderIndex set the leader index. -func (consensus *Consensus) SetLeaderIndex(f func(int) int) (current int) { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() - consensus.LeaderIndex = f(consensus.LeaderIndex) - return consensus.LeaderIndex -} - -func (consensus *Consensus) GetLeaderIndex() int { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() - return consensus.LeaderIndex -} - // SetViewChangingID set the current view change ID func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 9ab924c004..a36d3d9212 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -5,6 +5,7 @@ import ( "context" "encoding/hex" "fmt" + "math/big" "sync/atomic" "time" @@ -691,27 +692,42 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) prev := consensus.GetLeaderPubKey() - idx := consensus.SetLeaderIndex(func(i int) int { - if curBlockViewID%3 == 0 { - return i + 1 + if consensus.Blockchain.Config().IsLeaderRotation(blk.Epoch()) { + epochBlockViewID, err := consensus.getEpochFirstBlockViewID(blk.Epoch()) + if err != nil { + consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", blk.Epoch().Uint64()) + return + } + if epochBlockViewID > curBlockViewID { + consensus.getLogger().Error().Msg("[SetupForNewConsensus] Epoch block viewID is greater than current block viewID") + return } - return i - }) - pps := consensus.Decider.Participants() - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = &pps[idx%len(pps)] - fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %q\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) - consensus.pubKeyLock.Unlock() - if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { - // leader changed - go func() { - fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - consensus.ReadySignal <- SyncProposal - - }() + diff := curBlockViewID - epochBlockViewID + + pps := consensus.Decider.Participants() + idx := (int(diff) / 3) % len(pps) + consensus.pubKeyLock.Lock() + fmt.Println("(int(diff)/3)%len(pps) == ", idx) + consensus.LeaderPubKey = &pps[idx] + fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + consensus.pubKeyLock.Unlock() + if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { + // leader changed + go func() { + fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + consensus.ReadySignal <- SyncProposal + + }() + } + } else { + fmt.Printf("SetupForNewConsensus0 :%d future v%d new: %s prev: %s %v\n", utils.GetPort(), curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] + consensus.pubKeyLock.Unlock() } + // Update consensus keys at last so the change of leader status doesn't mess up normal flow if blk.IsLastBlockInEpoch() { consensus.SetMode(consensus.UpdateConsensusInformation()) @@ -720,6 +736,17 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.ResetState() } +func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, error) { + if epoch.Uint64() == 0 { + return 0, nil + } + epochBlock := consensus.Blockchain.GetBlockByNumber(epoch.Uint64() - 1) + if epochBlock == nil { + return 0, errors.Errorf("block not found for number %d", epoch.Uint64()-1) + } + return epochBlock.Header().ViewID().Uint64() + 1, nil +} + func (consensus *Consensus) postCatchup(initBN uint64) { if initBN < consensus.BlockNum() { consensus.getLogger().Info(). diff --git a/internal/params/config.go b/internal/params/config.go index d24f4be7b4..9e9a3b4fc5 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -262,6 +262,7 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, FeeCollectEpoch: big.NewInt(5), + LeaderRotationEpoch: big.NewInt(1), } // AllProtocolChanges ... @@ -301,6 +302,7 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch + big.NewInt(1), // LeaderRotationEpoch big.NewInt(0), // FeeCollectEpoch } @@ -341,6 +343,8 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch + // TODO place correct epoch number + big.NewInt(1), // LeaderRotationEpoch big.NewInt(0), // FeeCollectEpoch } @@ -481,6 +485,8 @@ type ChainConfig struct { // AllowlistEpoch is the first epoch to support allowlist of HIP18 AllowlistEpoch *big.Int + LeaderRotationEpoch *big.Int `json:"leader-rotation-epoch,omitempty"` + // FeeCollectEpoch is the first epoch that enables txn fees to be collected into the community-managed account. // It should >= StakingEpoch. // Before StakingEpoch, txn fees are paid to miner/leader. @@ -687,6 +693,10 @@ func (c *ChainConfig) IsAllowlistEpoch(epoch *big.Int) bool { return isForked(c.AllowlistEpoch, epoch) } +func (c *ChainConfig) IsLeaderRotation(epoch *big.Int) bool { + return isForked(c.LeaderRotationEpoch, epoch) +} + // IsFeeCollectEpoch determines whether Txn Fees will be collected into the community-managed account. func (c *ChainConfig) IsFeeCollectEpoch(epoch *big.Int) bool { return isForked(c.FeeCollectEpoch, epoch) From 61236819bcbe9560faa3147eefab3ad912d162e4 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 5 Sep 2022 22:41:06 +0700 Subject: [PATCH 150/420] update master --- api/service/explorer/service.go | 2 ++ consensus/consensus_v2.go | 9 +++------ consensus/view_change.go | 2 +- node/node_newblock.go | 1 - 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 5887bb3a02..1ebfa94ba1 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -223,6 +223,8 @@ func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { for i := cur; i > 0; i-- { block := s.blockchain.GetBlockByNumber(i) + leaderPubKey, _ := chain.GetLeaderPubKeyFromCoinbase(s.backend.Blockchain(), block.Header()) + w.Write([]byte(fmt.Sprintf("%s ", leaderPubKey.Bytes.Hex()))) w.Write([]byte(fmt.Sprintf("#%d ", i))) w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index a36d3d9212..cdd641f906 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -708,21 +708,18 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg pps := consensus.Decider.Participants() idx := (int(diff) / 3) % len(pps) consensus.pubKeyLock.Lock() - fmt.Println("(int(diff)/3)%len(pps) == ", idx) + //fmt.Println("(int(diff)/3)%len(pps) == ", idx) consensus.LeaderPubKey = &pps[idx] - fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + //fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) consensus.pubKeyLock.Unlock() if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { // leader changed go func() { - fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) consensus.ReadySignal <- SyncProposal - }() } } else { - fmt.Printf("SetupForNewConsensus0 :%d future v%d new: %s prev: %s %v\n", utils.GetPort(), curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + fmt.Printf("SetupForNewConsensus0 Before LeaderRotation") consensus.pubKeyLock.Lock() consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] consensus.pubKeyLock.Unlock() diff --git a/consensus/view_change.go b/consensus/view_change.go index 2936fba018..9c045a9eef 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -257,7 +257,7 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - fmt.Printf("Message to send leader111: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) + fmt.Printf("[startViewChange]: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } diff --git a/node/node_newblock.go b/node/node_newblock.go index 24e26ad99a..66bd4ed038 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -148,7 +148,6 @@ func (node *Node) ProposeNewBlock(commitSigs chan []byte) (*types.Block, error) if node.Blockchain().Config().IsStaking(header.Epoch()) { blsPubKeyBytes := leaderKey.Object.GetAddress() coinbase.SetBytes(blsPubKeyBytes[:]) - fmt.Println("coinbase.SetBytes leader: ", leaderKey.Bytes.Hex(), coinbase.Hex()) } emptyAddr := common.Address{} From a6d7bdaad177ade03422fba96831d865aece68d9 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 6 Sep 2022 02:58:48 +0700 Subject: [PATCH 151/420] fix leader --- consensus/consensus.go | 1 + consensus/consensus_service.go | 45 +++++++++++++++++++----- consensus/consensus_v2.go | 64 ++++++++++++++++++++++++++++------ internal/chain/engine.go | 2 +- 4 files changed, 91 insertions(+), 21 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index fb7bf79b53..f2f995ae7d 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -76,6 +76,7 @@ type Consensus struct { // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 + epoch uint64 // Blockhash - 32 byte blockHash [32]byte // Block to run consensus on diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index d26ea8ced5..d2747d19d9 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -77,6 +77,7 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) + consensus.pubKeyLock.Unlock() consensus.getLogger().Info().Msg("My Committee updated") for i := range pubKeys { consensus.getLogger().Info(). @@ -84,17 +85,24 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi Str("BLSPubKey", pubKeys[i].Bytes.Hex()). Msg("Member") } - - allKeys := consensus.Decider.Participants() - if len(allKeys) != 0 { - consensus.LeaderPubKey = &allKeys[0] - consensus.getLogger().Info(). - Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") + if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { + consensus.updateLeader() } else { - consensus.getLogger().Error(). - Msg("[UpdatePublicKeys] Participants is empty") + consensus.pubKeyLock.Lock() + allKeys := consensus.Decider.Participants() + consensus.pubKeyLock.Unlock() + if len(allKeys) != 0 { + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = &allKeys[0] + consensus.pubKeyLock.Unlock() + consensus.getLogger().Info(). + Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") + } else { + consensus.getLogger().Error(). + Msg("[UpdatePublicKeys] Participants is empty") + } } - consensus.pubKeyLock.Unlock() + // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. consensus.UpdateBitmaps() @@ -474,6 +482,15 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } +func (consensus *Consensus) SetCurEpoch(epoch uint64) { + fmt.Println("SetCurEpoch", epoch) + atomic.StoreUint64(&consensus.epoch, epoch) +} + +func (consensus *Consensus) GetCurEpoch() *big.Int { + return big.NewInt(int64(atomic.LoadUint64(&consensus.epoch))) +} + // SetViewChangingID set the current view change ID func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) @@ -608,3 +625,13 @@ func (consensus *Consensus) getLogger() *zerolog.Logger { Logger() return &logger } + +func UpdatePublicKeyDefault(consensus *Consensus) { + if allKeys := consensus.Decider.Participants(); len(allKeys) > 0 { + consensus.LeaderPubKey = &allKeys[0] + } +} + +func UpdatePublicKeyRotate(consensus *Consensus) { + //consensus +} diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index cdd641f906..1313f279be 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -11,7 +11,6 @@ import ( bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" - nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" @@ -687,15 +686,14 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess return nil } -// SetupForNewConsensus sets the state for new consensus -func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { - atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) - curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) +func (consensus *Consensus) updateLeader() { + curBlockViewID := consensus.GetCurBlockViewID() prev := consensus.GetLeaderPubKey() - if consensus.Blockchain.Config().IsLeaderRotation(blk.Epoch()) { - epochBlockViewID, err := consensus.getEpochFirstBlockViewID(blk.Epoch()) + epoch := consensus.GetCurEpoch() + if consensus.Blockchain.Config().IsLeaderRotation(epoch) { + epochBlockViewID, err := consensus.getEpochFirstBlockViewID(epoch) if err != nil { - consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", blk.Epoch().Uint64()) + consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", epoch) return } if epochBlockViewID > curBlockViewID { @@ -706,7 +704,8 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg diff := curBlockViewID - epochBlockViewID pps := consensus.Decider.Participants() - idx := (int(diff) / 3) % len(pps) + fmt.Println("diff ", diff, "epochBlockViewID: ", curBlockViewID, "epochBlockViewID", epochBlockViewID, (int(diff) / 5), len(pps), epoch) + idx := (int(diff) / 5) % len(pps) consensus.pubKeyLock.Lock() //fmt.Println("(int(diff)/3)%len(pps) == ", idx) consensus.LeaderPubKey = &pps[idx] @@ -718,8 +717,51 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.ReadySignal <- SyncProposal }() } + } +} + +// SetupForNewConsensus sets the state for new consensus +func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { + atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) + consensus.SetCurBlockViewID(committedMsg.ViewID + 1) + if blk.IsLastBlockInEpoch() { + consensus.SetCurEpoch(blk.Epoch().Uint64() + 1) + } else { + consensus.SetCurEpoch(blk.Epoch().Uint64()) + } + //prev := consensus.GetLeaderPubKey() + if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { + //consensus.updateLeader() + consensus.updateLeader() + + /*{ + epochBlockViewID, err := consensus.getEpochFirstBlockViewID(consensus.GetCurEpoch()) + if err != nil { + consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", blk.Epoch().Uint64()) + return + } + if epochBlockViewID > curBlockViewID { + consensus.getLogger().Error().Msg("[SetupForNewConsensus] Epoch block viewID is greater than current block viewID") + return + } + + diff := curBlockViewID - epochBlockViewID + + pps := consensus.Decider.Participants() + idx := (int(diff) / 3) % len(pps) + consensus.pubKeyLock.Lock() + //fmt.Println("(int(diff)/3)%len(pps) == ", idx) + consensus.LeaderPubKey = &pps[idx] + //fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + consensus.pubKeyLock.Unlock() + if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { + // leader changed + go func() { + consensus.ReadySignal <- SyncProposal + }() + } + }*/ } else { - fmt.Printf("SetupForNewConsensus0 Before LeaderRotation") consensus.pubKeyLock.Lock() consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] consensus.pubKeyLock.Unlock() @@ -737,7 +779,7 @@ func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, er if epoch.Uint64() == 0 { return 0, nil } - epochBlock := consensus.Blockchain.GetBlockByNumber(epoch.Uint64() - 1) + epochBlock := consensus.Blockchain.GetBlockByNumber(shard.Schedule.EpochLastBlock(epoch.Uint64() - 1)) if epochBlock == nil { return 0, errors.Errorf("block not found for number %d", epoch.Uint64()-1) } diff --git a/internal/chain/engine.go b/internal/chain/engine.go index a340e9c278..6e018dccfd 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -186,7 +186,7 @@ func (e *engineImpl) VerifyVRF( return nil } -// retrieve corresponding blsPublicKey from Coinbase Address +// GetLeaderPubKeyFromCoinbase retrieve corresponding blsPublicKey from Coinbase Address func GetLeaderPubKeyFromCoinbase( blockchain engine.ChainReader, h *block.Header, ) (*bls.PublicKeyWrapper, error) { From be0f339930712d9bb4857c5c8504a7f0ff8c0bc4 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 7 Sep 2022 22:20:05 +0700 Subject: [PATCH 152/420] check leader for N blocks --- consensus/consensus.go | 5 +++ consensus/consensus_service.go | 29 +++++++-------- consensus/consensus_v2.go | 66 ++++++++++++++++++++-------------- consensus/view_change.go | 2 +- 4 files changed, 59 insertions(+), 43 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index f2f995ae7d..f9b48be2ab 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -181,6 +181,11 @@ func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper { defer consensus.pubKeyLock.Unlock() return consensus.LeaderPubKey } +func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = pub + consensus.pubKeyLock.Unlock() +} func (consensus *Consensus) GetPrivateKeys() multibls.PrivateKeys { return consensus.priKey diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index d2747d19d9..0d7a6c5543 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -77,8 +77,20 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) + allKeys := consensus.Decider.Participants() consensus.pubKeyLock.Unlock() - consensus.getLogger().Info().Msg("My Committee updated") + if len(allKeys) != 0 { + first := consensus.Decider.FirstParticipant( + shard.Schedule.InstanceForEpoch(consensus.Blockchain.CurrentHeader().Epoch())) + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = first + consensus.pubKeyLock.Unlock() + consensus.getLogger().Info(). + Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") + } else { + consensus.getLogger().Error(). + Msg("[UpdatePublicKeys] Participants is empty") + } for i := range pubKeys { consensus.getLogger().Info(). Int("index", i). @@ -87,20 +99,6 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi } if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { consensus.updateLeader() - } else { - consensus.pubKeyLock.Lock() - allKeys := consensus.Decider.Participants() - consensus.pubKeyLock.Unlock() - if len(allKeys) != 0 { - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = &allKeys[0] - consensus.pubKeyLock.Unlock() - consensus.getLogger().Info(). - Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") - } else { - consensus.getLogger().Error(). - Msg("[UpdatePublicKeys] Participants is empty") - } } // reset states after update public keys @@ -483,7 +481,6 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { } func (consensus *Consensus) SetCurEpoch(epoch uint64) { - fmt.Println("SetCurEpoch", epoch) atomic.StoreUint64(&consensus.epoch, epoch) } diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 1313f279be..d3e8dc8eb7 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -4,13 +4,13 @@ import ( "bytes" "context" "encoding/hex" - "fmt" "math/big" "sync/atomic" "time" bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" + "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" @@ -393,7 +393,6 @@ func (consensus *Consensus) Start( consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() case newBlock := <-blockChannel: - //consensus.ReshardingNextLeader(newBlock) consensus.getLogger().Info(). Uint64("MsgBlockNum", newBlock.NumberU64()). Msg("[ConsensusMainLoop] Received Proposed New Block!") @@ -405,7 +404,6 @@ func (consensus *Consensus) Start( } // Sleep to wait for the full block time consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") - <-time.After(time.Until(consensus.NextBlockDue)) consensus.StartFinalityCount() @@ -530,7 +528,6 @@ func (consensus *Consensus) getLastMileBlocksAndMsg(bnStart uint64) ([]*types.Bl // preCommitAndPropose commit the current block with 67% commit signatures and start // proposing new block which will wait on the full commit signatures to finish func (consensus *Consensus) preCommitAndPropose(blk *types.Block) error { - //fmt.Println("preCommitAndPropose", utils.GetPort(), blk.NumberU64()) if blk == nil { return errors.New("block to pre-commit is nil") } @@ -687,30 +684,48 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess } func (consensus *Consensus) updateLeader() { - curBlockViewID := consensus.GetCurBlockViewID() prev := consensus.GetLeaderPubKey() epoch := consensus.GetCurEpoch() + curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() if consensus.Blockchain.Config().IsLeaderRotation(epoch) { - epochBlockViewID, err := consensus.getEpochFirstBlockViewID(epoch) - if err != nil { - consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", epoch) - return + leader := consensus.GetLeaderPubKey() + for i := uint64(0); i < 5; i++ { + header := consensus.Blockchain.GetHeaderByNumber(curNumber - i) + if header == nil { + return + } + // Previous epoch, we should not change leader. + if header.Epoch().Uint64() != epoch.Uint64() { + return + } + // Check if the same leader. + pub, err := chain.GetLeaderPubKeyFromCoinbase(consensus.Blockchain, header) + if err != nil { + utils.Logger().Error().Err(err).Msg("Failed to get leader public key from coinbase") + return + } + if !pub.Object.IsEqual(leader.Object) { + // Another leader. + return + } + } + // Passed all checks, we can change leader. + var ( + wasFound bool + next *bls.PublicKeyWrapper + ) + // The same leader for N blocks. + if consensus.Blockchain.Config().IsAllowlistEpoch(epoch) { + wasFound, next = consensus.Decider.NthNextHmyExt(shard.Schedule.InstanceForEpoch(epoch), leader, 1) + } else { + wasFound, next = consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) } - if epochBlockViewID > curBlockViewID { - consensus.getLogger().Error().Msg("[SetupForNewConsensus] Epoch block viewID is greater than current block viewID") + if !wasFound { + utils.Logger().Error().Msg("Failed to get next leader") return + } else { + consensus.SetLeaderPubKey(next) } - - diff := curBlockViewID - epochBlockViewID - - pps := consensus.Decider.Participants() - fmt.Println("diff ", diff, "epochBlockViewID: ", curBlockViewID, "epochBlockViewID", epochBlockViewID, (int(diff) / 5), len(pps), epoch) - idx := (int(diff) / 5) % len(pps) - consensus.pubKeyLock.Lock() - //fmt.Println("(int(diff)/3)%len(pps) == ", idx) - consensus.LeaderPubKey = &pps[idx] - //fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) - consensus.pubKeyLock.Unlock() if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { // leader changed go func() { @@ -729,6 +744,9 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg } else { consensus.SetCurEpoch(blk.Epoch().Uint64()) } + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] + consensus.pubKeyLock.Unlock() //prev := consensus.GetLeaderPubKey() if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { //consensus.updateLeader() @@ -761,10 +779,6 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg }() } }*/ - } else { - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] - consensus.pubKeyLock.Unlock() } // Update consensus keys at last so the change of leader status doesn't mess up normal flow diff --git a/consensus/view_change.go b/consensus/view_change.go index 9c045a9eef..55d3d56a8b 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -176,7 +176,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe cur := consensus.GetCurBlockViewID() if viewID > cur { - gap = int(viewID - consensus.GetCurBlockViewID()) + gap = int(viewID - cur) } var lastLeaderPubKey *bls.PublicKeyWrapper var err error From ca7a4f03397d41db768e51076ae35cf5e33babf3 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sat, 10 Sep 2022 18:02:45 +0700 Subject: [PATCH 153/420] fix --- api/service/explorer/service.go | 2 -- cmd/harmony/main.go | 3 --- consensus/consensus_service.go | 21 --------------------- consensus/quorum/quorum.go | 7 +------ 4 files changed, 1 insertion(+), 32 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 1ebfa94ba1..b13f73db61 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -115,8 +115,6 @@ func (s *Service) Run() *http.Server { s.router = mux.NewRouter() - fmt.Println("++", addr) - // Set up router for addresses. // Fetch addresses request, accepts parameter size: how much addresses to read, // parameter prefix: from which address prefix start diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 75eabfb2e8..1dabf5c459 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -265,8 +265,6 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { var err error - fmt.Println("OS: ", os.Args) - nodeconfigSetShardSchedule(hc) nodeconfig.SetShardingSchedule(shard.Schedule) nodeconfig.SetVersion(getHarmonyVersion()) @@ -787,7 +785,6 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // Set the consensus ID to be the current block number viewID := currentNode.Blockchain().CurrentBlock().Header().ViewID().Uint64() - fmt.Println("viewID:", viewID) currentConsensus.SetViewIDs(viewID + 1) utils.Logger().Info(). Uint64("viewID", viewID). diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 0d7a6c5543..96056b5231 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -219,7 +219,6 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } - fmt.Println("[checkViewID] Set LEADEER PUB KEY ", msg.SenderPubkeys[0].Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() @@ -403,7 +402,6 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { Str("leaderPubKey", leaderPubKey.Bytes.Hex()). Msg("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain") consensus.pubKeyLock.Lock() - fmt.Println("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain", leaderPubKey.Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = leaderPubKey consensus.pubKeyLock.Unlock() } @@ -498,15 +496,6 @@ func (consensus *Consensus) StartFinalityCount() { consensus.finalityCounter.Store(time.Now().UnixNano()) } -//func (consensus *Consensus) ReshardingNextLeader(newblock *types.Block) { -// consensus.pubKeyLock.Lock() -// fmt.Println("nextBlock1 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) -// consensus.LeaderPubKey = consensus.getNextLeaderKey(consensus.GetCurBlockViewID() + 1) -// fmt.Println("nextBlock2 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) -// consensus.pubKeyLock.Unlock() -// -//} - // FinishFinalityCount calculate the current finality func (consensus *Consensus) FinishFinalityCount() { d := time.Now().UnixNano() @@ -622,13 +611,3 @@ func (consensus *Consensus) getLogger() *zerolog.Logger { Logger() return &logger } - -func UpdatePublicKeyDefault(consensus *Consensus) { - if allKeys := consensus.Decider.Participants(); len(allKeys) > 0 { - consensus.LeaderPubKey = &allKeys[0] - } -} - -func UpdatePublicKeyRotate(consensus *Consensus) { - //consensus -} diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 7e76dfcc6b..867fd99677 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -231,17 +231,12 @@ func (s *cIdentities) NthNextHmy(instance shardingconfig.Instance, pubKey *bls.P Msg("[NthNextHmy] pubKey not found") } numNodes := instance.NumHarmonyOperatedNodesPerShard() - //fmt.Println("??idx:", idx, numNodes) // sanity check to avoid out of bound access if numNodes <= 0 || numNodes > len(s.publicKeys) { numNodes = len(s.publicKeys) } idx = (idx + next) % numNodes - //fmt.Println("-------idx:", idx) - - new := &s.publicKeys[idx] - fmt.Println("NthNextHmy: ", pubKey.Bytes.Hex(), new.Bytes.Hex()) - return found, new + return found, &s.publicKeys[idx] } // NthNextHmyExt return the Nth next pubkey of Harmony + allowlist nodes, next can be negative number From f96004cfe866f67850410e078f1d1b1616b07125 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 14:04:09 +0700 Subject: [PATCH 154/420] fix --- consensus/consensus_service.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 96056b5231..25594e044e 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync/atomic" "time" @@ -468,7 +467,6 @@ func (consensus *Consensus) isLeader() bool { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { - fmt.Println("SetViewIDs", height) consensus.SetCurBlockViewID(height) consensus.SetViewChangingID(height) } From 3e7ceddb5602159af9b0dbf094aead3d19b8fbca Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 15:27:56 +0700 Subject: [PATCH 155/420] Cleanup and fix update pub keys. --- consensus/consensus_service.go | 31 +++++++++--------- consensus/consensus_v2.go | 35 ++------------------- consensus/leader.go | 11 ++----- consensus/quorum/quorum.go | 1 - consensus/view_change.go | 15 +-------- p2p/stream/common/streammanager/cooldown.go | 6 ++++ 6 files changed, 27 insertions(+), 72 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 25594e044e..77996258e4 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync/atomic" "time" @@ -73,33 +74,31 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { + if utils.GetPort() == 9000 { + //utils.Logger().Info().Msg("UpdatePublicKeys") + fmt.Println("UpdatePublicKeys", len(pubKeys), len(allowlist)) + } // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) + consensus.getLogger().Info().Msg("My Committee updated") + for i := range pubKeys { + consensus.getLogger().Info(). + Int("index", i). + Str("BLSPubKey", pubKeys[i].Bytes.Hex()). + Msg("Member") + } + allKeys := consensus.Decider.Participants() - consensus.pubKeyLock.Unlock() if len(allKeys) != 0 { - first := consensus.Decider.FirstParticipant( - shard.Schedule.InstanceForEpoch(consensus.Blockchain.CurrentHeader().Epoch())) - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = first - consensus.pubKeyLock.Unlock() + consensus.LeaderPubKey = &allKeys[0] consensus.getLogger().Info(). Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") } else { consensus.getLogger().Error(). Msg("[UpdatePublicKeys] Participants is empty") } - for i := range pubKeys { - consensus.getLogger().Info(). - Int("index", i). - Str("BLSPubKey", pubKeys[i].Bytes.Hex()). - Msg("Member") - } - if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { - consensus.updateLeader() - } - + consensus.pubKeyLock.Unlock() // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. consensus.UpdateBitmaps() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index d3e8dc8eb7..fe685ca857 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -645,7 +645,6 @@ func (consensus *Consensus) tryCatchup() error { consensus.getLogger().Error().Err(err).Msg("[TryCatchup] Failed to add block to chain") return err } - //fmt.Println("tryCatchup ", utils.GetPort(), blk.NumberU64()) select { // TODO: Remove this when removing dns sync and stream sync is fully up case consensus.VerifiedNewBlock <- blk: @@ -683,7 +682,7 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess return nil } -func (consensus *Consensus) updateLeader() { +func (consensus *Consensus) rotateLeader() { prev := consensus.GetLeaderPubKey() epoch := consensus.GetCurEpoch() curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() @@ -747,38 +746,8 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.pubKeyLock.Lock() consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] consensus.pubKeyLock.Unlock() - //prev := consensus.GetLeaderPubKey() if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { - //consensus.updateLeader() - consensus.updateLeader() - - /*{ - epochBlockViewID, err := consensus.getEpochFirstBlockViewID(consensus.GetCurEpoch()) - if err != nil { - consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", blk.Epoch().Uint64()) - return - } - if epochBlockViewID > curBlockViewID { - consensus.getLogger().Error().Msg("[SetupForNewConsensus] Epoch block viewID is greater than current block viewID") - return - } - - diff := curBlockViewID - epochBlockViewID - - pps := consensus.Decider.Participants() - idx := (int(diff) / 3) % len(pps) - consensus.pubKeyLock.Lock() - //fmt.Println("(int(diff)/3)%len(pps) == ", idx) - consensus.LeaderPubKey = &pps[idx] - //fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) - consensus.pubKeyLock.Unlock() - if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { - // leader changed - go func() { - consensus.ReadySignal <- SyncProposal - }() - } - }*/ + consensus.rotateLeader() } // Update consensus keys at last so the change of leader status doesn't mess up normal flow diff --git a/consensus/leader.go b/consensus/leader.go index a359c229a4..4f72ae6268 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -200,16 +200,12 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { // TODO HERE - //if recvMsg.ViewID == 10 { - // return - //} + if recvMsg.ViewID == 21 { + return + } consensus.mutex.Lock() defer consensus.mutex.Unlock() //// Read - Start - if consensus.ShardID == 0 { - //fmt.Println("onCommit ", recvMsg.BlockNum) - } - if !consensus.isRightBlockNumAndViewID(recvMsg) { return } @@ -341,5 +337,4 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED) } - //fmt.Println("onCommit99: ", utils.GetPort(), recvMsg.BlockNum) } diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 867fd99677..6cf79e549a 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -75,7 +75,6 @@ type ParticipantTracker interface { Participants() multibls.PublicKeys IndexOf(bls.SerializedPublicKey) int ParticipantsCount() int64 - NthNext(*bls.PublicKeyWrapper, int) (bool, *bls.PublicKeyWrapper) NthNextHmy(shardingconfig.Instance, *bls.PublicKeyWrapper, int) (bool, *bls.PublicKeyWrapper) NthNextHmyExt(shardingconfig.Instance, *bls.PublicKeyWrapper, int) (bool, *bls.PublicKeyWrapper) FirstParticipant(shardingconfig.Instance) *bls.PublicKeyWrapper diff --git a/consensus/view_change.go b/consensus/view_change.go index 55d3d56a8b..3bd0c181f2 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync" "time" @@ -162,7 +161,6 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { Uint64("stuckBlockViewID", stuckBlockViewID). Msg("[getNextViewID]") - fmt.Println("end getNextViewID: ", nextViewID, viewChangeDuration) // duration is always the fixed view change duration for synchronous view change return nextViewID, viewChangeDuration } @@ -235,7 +233,6 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe lastLeaderPubKey, gap) } - fmt.Println("wasfoundNext", consensus.Blockchain.Config().IsAllowlistEpoch(epoch), wasFound, next.Bytes.Hex(), lastLeaderPubKey.Bytes.Hex()) if !wasFound { consensus.getLogger().Warn(). Str("key", consensus.LeaderPubKey.Bytes.Hex()). @@ -257,7 +254,6 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - fmt.Printf("[startViewChange]: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } @@ -268,7 +264,6 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() - //fmt.Println("startViewChange", nextViewID) consensus.SetViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet @@ -279,8 +274,6 @@ func (consensus *Consensus) startViewChange() { consensus.pubKeyLock.Lock() lpk := consensus.getNextLeaderKey(nextViewID) consensus.LeaderPubKey = lpk - //fmt.Println("Message to send leader cur: ", consensus.LeaderPubKey.Bytes.Hex(), "next: ", lpk.Bytes.Hex()) - //fmt.Println("Message to send leader: ", consensus.LeaderPubKey.Bytes.Hex()) consensus.pubKeyLock.Unlock() consensus.getLogger().Warn(). @@ -314,9 +307,7 @@ func (consensus *Consensus) startViewChange() { if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { continue } - // Тут уже другой leader msgToSend := consensus.constructViewChangeMessage(&key) - fmt.Println("Message to send leader222: ", consensus.LeaderPubKey.Bytes.Hex()) if err := consensus.msgSender.SendWithRetry( consensus.BlockNum(), msg_pb.MessageType_VIEWCHANGE, @@ -374,8 +365,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.ResetState() } - fmt.Println("[startNewView]", newLeaderPriKey.Pub.Bytes.Hex()) - consensus.LeaderPubKey = newLeaderPriKey.Pub + consensus.SetLeaderPubKey(newLeaderPriKey.Pub) return nil } @@ -492,8 +482,6 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.mutex.Lock() defer consensus.mutex.Unlock() - fmt.Printf("[onNewView] received new view message from %+v\n", recvMsg) - consensus.getLogger().Info(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -582,7 +570,6 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // newView message verified success, override my state consensus.SetViewIDs(recvMsg.ViewID) consensus.pubKeyLock.Lock() - fmt.Println("[onNewView1221] new leader key cur:", consensus.LeaderPubKey.Bytes.Hex(), " new: ", senderKey.Bytes.Hex()) consensus.LeaderPubKey = senderKey consensus.pubKeyLock.Unlock() consensus.ResetViewChangeState() diff --git a/p2p/stream/common/streammanager/cooldown.go b/p2p/stream/common/streammanager/cooldown.go index 0f837c01de..d8b58346c0 100644 --- a/p2p/stream/common/streammanager/cooldown.go +++ b/p2p/stream/common/streammanager/cooldown.go @@ -2,6 +2,7 @@ package streammanager import ( "container/list" + "sync" "time" "github.com/libp2p/go-libp2p/core/peer" @@ -14,6 +15,7 @@ const ( ) type coolDownCache struct { + mu sync.Mutex timeCache *timecache.TimeCache } @@ -26,6 +28,8 @@ func newCoolDownCache() *coolDownCache { // Has check and add the peer ID to the cache func (cache *coolDownCache) Has(id peer.ID) bool { + cache.mu.Lock() + defer cache.mu.Unlock() has := cache.timeCache.Has(string(id)) if !has { cache.timeCache.Add(string(id)) @@ -35,6 +39,8 @@ func (cache *coolDownCache) Has(id peer.ID) bool { // Reset the cool down cache func (cache *coolDownCache) Reset() { + cache.mu.Lock() + defer cache.mu.Unlock() cache.timeCache.Q = list.New() cache.timeCache.M = make(map[string]time.Time) } From 84b19ac9fb1830c9a8a293f68f2fb39a26a539a4 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 16:32:36 +0700 Subject: [PATCH 156/420] Rotate leader. --- consensus/consensus.go | 1 - consensus/consensus_service.go | 8 -------- consensus/consensus_v2.go | 12 +++--------- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index f9b48be2ab..1259576834 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -76,7 +76,6 @@ type Consensus struct { // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 - epoch uint64 // Blockhash - 32 byte blockHash [32]byte // Block to run consensus on diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 77996258e4..af344c6820 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -475,14 +475,6 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } -func (consensus *Consensus) SetCurEpoch(epoch uint64) { - atomic.StoreUint64(&consensus.epoch, epoch) -} - -func (consensus *Consensus) GetCurEpoch() *big.Int { - return big.NewInt(int64(atomic.LoadUint64(&consensus.epoch))) -} - // SetViewChangingID set the current view change ID func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index fe685ca857..7c4290fad2 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -682,9 +682,8 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess return nil } -func (consensus *Consensus) rotateLeader() { +func (consensus *Consensus) rotateLeader(epoch *big.Int) { prev := consensus.GetLeaderPubKey() - epoch := consensus.GetCurEpoch() curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() if consensus.Blockchain.Config().IsLeaderRotation(epoch) { leader := consensus.GetLeaderPubKey() @@ -738,16 +737,11 @@ func (consensus *Consensus) rotateLeader() { func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) consensus.SetCurBlockViewID(committedMsg.ViewID + 1) - if blk.IsLastBlockInEpoch() { - consensus.SetCurEpoch(blk.Epoch().Uint64() + 1) - } else { - consensus.SetCurEpoch(blk.Epoch().Uint64()) - } consensus.pubKeyLock.Lock() consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] consensus.pubKeyLock.Unlock() - if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { - consensus.rotateLeader() + if consensus.Blockchain.Config().IsLeaderRotation(blk.Epoch()) { + consensus.rotateLeader(blk.Epoch()) } // Update consensus keys at last so the change of leader status doesn't mess up normal flow From 9cb2162218cd783c327245e97ee0b4e9a9f631ec Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 13 Sep 2022 01:10:13 +0700 Subject: [PATCH 157/420] fix fix fix fix fix --- consensus/consensus_service.go | 5 ----- consensus/consensus_v2.go | 16 ++++++++++++---- consensus/leader.go | 4 ---- internal/params/config.go | 7 ++++++- node/node_newblock.go | 2 +- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index af344c6820..e6be606f2e 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync/atomic" "time" @@ -74,10 +73,6 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { - if utils.GetPort() == 9000 { - //utils.Logger().Info().Msg("UpdatePublicKeys") - fmt.Println("UpdatePublicKeys", len(pubKeys), len(allowlist)) - } // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 7c4290fad2..3122a368b9 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -8,6 +8,7 @@ import ( "sync/atomic" "time" + "github.com/ethereum/go-ethereum/common" bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" "github.com/harmony-one/harmony/internal/chain" @@ -685,10 +686,11 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess func (consensus *Consensus) rotateLeader(epoch *big.Int) { prev := consensus.GetLeaderPubKey() curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() + utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), consensus.Blockchain.Config().IsLeaderRotation(epoch), consensus.Blockchain.Config().LeaderRotationBlocksCount) if consensus.Blockchain.Config().IsLeaderRotation(epoch) { leader := consensus.GetLeaderPubKey() - for i := uint64(0); i < 5; i++ { - header := consensus.Blockchain.GetHeaderByNumber(curNumber - i) + for i := 0; i < consensus.Blockchain.Config().LeaderRotationBlocksCount; i++ { + header := consensus.Blockchain.GetHeaderByNumber(curNumber - uint64(i)) if header == nil { return } @@ -740,8 +742,14 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.pubKeyLock.Lock() consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] consensus.pubKeyLock.Unlock() - if consensus.Blockchain.Config().IsLeaderRotation(blk.Epoch()) { - consensus.rotateLeader(blk.Epoch()) + var epoch *big.Int + if blk.IsLastBlockInEpoch() { + epoch = new(big.Int).Add(blk.Epoch(), common.Big1) + } else { + epoch = blk.Epoch() + } + if consensus.Blockchain.Config().IsLeaderRotation(epoch) { + consensus.rotateLeader(epoch) } // Update consensus keys at last so the change of leader status doesn't mess up normal flow diff --git a/consensus/leader.go b/consensus/leader.go index 4f72ae6268..422508762b 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -199,10 +199,6 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { - // TODO HERE - if recvMsg.ViewID == 21 { - return - } consensus.mutex.Lock() defer consensus.mutex.Unlock() //// Read - Start diff --git a/internal/params/config.go b/internal/params/config.go index 9e9a3b4fc5..55e8126e80 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -262,7 +262,8 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, FeeCollectEpoch: big.NewInt(5), - LeaderRotationEpoch: big.NewInt(1), + LeaderRotationEpoch: big.NewInt(4), + LeaderRotationBlocksCount: 5, } // AllProtocolChanges ... @@ -303,6 +304,7 @@ var ( big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } @@ -345,6 +347,7 @@ var ( big.NewInt(0), // AllowlistEpoch // TODO place correct epoch number big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } @@ -487,6 +490,8 @@ type ChainConfig struct { LeaderRotationEpoch *big.Int `json:"leader-rotation-epoch,omitempty"` + LeaderRotationBlocksCount int `json:"leader-rotation-blocks-count,omitempty"` + // FeeCollectEpoch is the first epoch that enables txn fees to be collected into the community-managed account. // It should >= StakingEpoch. // Before StakingEpoch, txn fees are paid to miner/leader. diff --git a/node/node_newblock.go b/node/node_newblock.go index 66bd4ed038..8738a98076 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -89,7 +89,7 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp newBlock, err := node.ProposeNewBlock(newCommitSigsChan) if err == nil { - fmt.Printf("ProposeNewBlock: #%d :%d @%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), newBlock.Header().ViewID().Int64(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) + fmt.Printf("ProposeNewBlock: #%d :%d e:%d, sh:%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), node.Blockchain().CurrentHeader().Epoch().Uint64(), node.Blockchain().ShardID(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). From b7d4bb6c92960b820aa3eb26aad357143e69eadc Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 13 Sep 2022 02:15:25 +0700 Subject: [PATCH 158/420] Cleaned. --- api/service/explorer/service.go | 48 ------------------------------- cmd/harmony/main.go | 3 +- consensus/consensus_v2.go | 13 --------- consensus/validator.go | 14 --------- consensus/view_change.go | 8 ------ node/node_newblock.go | 3 -- test/configs/local-resharding.txt | 31 ++++++++++++++------ test/deploy.sh | 5 +--- 8 files changed, 24 insertions(+), 101 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index b13f73db61..b5ffa1b8b0 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -7,13 +7,11 @@ import ( "fmt" "net" "net/http" - "os" "path" "strconv" "time" "github.com/RoaringBitmap/roaring/roaring64" - "github.com/harmony-one/harmony/internal/common" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" ethCommon "github.com/ethereum/go-ethereum/common" @@ -121,9 +119,6 @@ func (s *Service) Run() *http.Server { s.router.Path("/addresses").Queries("size", "{[0-9]*?}", "prefix", "{[a-zA-Z0-9]*?}").HandlerFunc(s.GetAddresses).Methods("GET") s.router.Path("/addresses").HandlerFunc(s.GetAddresses) s.router.Path("/height").HandlerFunc(s.GetHeight) - s.router.Path("/leader").HandlerFunc(s.GetLeader) - s.router.Path("/blocks").HandlerFunc(s.GetBlocks) - s.router.Path("/halt").HandlerFunc(s.halt) // Set up router for supply info s.router.Path("/burn-addresses").Queries().HandlerFunc(s.GetInaccessibleAddressInfo).Methods("GET") @@ -191,49 +186,6 @@ type HeightResponse struct { S3 uint64 `json:"3,omitempty"` } -func (s *Service) GetLeader(w http.ResponseWriter, r *http.Request) { - if s.backend.IsCurrentlyLeader() { - w.Write([]byte("true ")) - } else { - w.Write([]byte("false")) - } - - keys := "" - for _, p := range s.backend.GetPublicKeys() { - addr := common.Address{} - addrBytes := p.Object.GetAddress() - addr.SetBytes(addrBytes[:]) - keys += fmt.Sprintf("%s ", addr.String()) - break - } - //blsPubKeyBytes := leaderKey.Object.GetAddress() - //coinbase.SetBytes(blsPubKeyBytes[:]) - - w.Write([]byte(fmt.Sprintf(" %d", s.blockchain.ShardID()))) - w.Write([]byte(fmt.Sprintf(" %s", s.Port))) - w.Write([]byte(fmt.Sprintf(" %s", keys))) - w.Write([]byte(fmt.Sprintf(" %s", s.backend.GetPublicKeys().SerializeToHexStr()))) - -} - -func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { - cur := s.blockchain.CurrentHeader().Number().Uint64() - - for i := cur; i > 0; i-- { - block := s.blockchain.GetBlockByNumber(i) - leaderPubKey, _ := chain.GetLeaderPubKeyFromCoinbase(s.backend.Blockchain(), block.Header()) - w.Write([]byte(fmt.Sprintf("%s ", leaderPubKey.Bytes.Hex()))) - w.Write([]byte(fmt.Sprintf("#%d ", i))) - w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) - w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) - w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) - } -} - -func (s *Service) halt(w http.ResponseWriter, r *http.Request) { - os.Exit(0) -} - // GetHeight returns heights of current and beacon chains if needed. func (s *Service) GetHeight(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 1dabf5c459..437a91dc59 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -420,9 +420,8 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { if currentNode.NodeConfig.Role() == nodeconfig.Validator { currentNode.RegisterValidatorServices() } else if currentNode.NodeConfig.Role() == nodeconfig.ExplorerNode { - + currentNode.RegisterExplorerServices() } - currentNode.RegisterExplorerServices() currentNode.RegisterService(service.CrosslinkSending, crosslink_sending.New(currentNode, currentNode.Blockchain())) if hc.Pprof.Enabled { setupPprofService(currentNode, hc) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 3122a368b9..e87ea21012 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -660,8 +660,6 @@ func (consensus *Consensus) tryCatchup() error { } func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMessage) error { - // this function evaluates for all, leader and validators. - if consensus.Blockchain().CurrentBlock().NumberU64() < blk.NumberU64() { if _, err := consensus.Blockchain().InsertChain([]*types.Block{blk}, !consensus.FBFTLog.IsBlockVerified(blk.Hash())); err != nil { consensus.getLogger().Error().Err(err).Msg("[commitBlock] Failed to add block to chain") @@ -760,17 +758,6 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.ResetState() } -func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, error) { - if epoch.Uint64() == 0 { - return 0, nil - } - epochBlock := consensus.Blockchain.GetBlockByNumber(shard.Schedule.EpochLastBlock(epoch.Uint64() - 1)) - if epochBlock == nil { - return 0, errors.Errorf("block not found for number %d", epoch.Uint64()-1) - } - return epochBlock.Header().ViewID().Uint64() + 1, nil -} - func (consensus *Consensus) postCatchup(initBN uint64) { if initBN < consensus.BlockNum() { consensus.getLogger().Info(). diff --git a/consensus/validator.go b/consensus/validator.go index 013769395c..a73ac92eb3 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -18,7 +18,6 @@ import ( ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { - recvMsg, err := consensus.ParseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). @@ -27,9 +26,6 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Msg("[OnAnnounce] Unparseable leader message") return } - if consensus.ShardID == 0 { - //fmt.Println("onAnnounce called ", recvMsg.BlockNum) - } // NOTE let it handle its own logs if !consensus.onAnnounceSanityChecks(recvMsg) { @@ -188,9 +184,6 @@ func (consensus *Consensus) sendCommitMessages(blockObj *types.Block) { // if onPrepared accepts the prepared message from the leader, then // it will send a COMMIT message for the leader to receive on the network. func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { - if consensus.ShardID == 0 { - //fmt.Println("onPrepared", recvMsg.BlockNum) - } consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -406,13 +399,6 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { consensus.getLogger().Info().Msg("[OnCommitted] Start consensus timer (new block added)") consensus.consensusTimeout[timeoutConsensus].Start() } - - //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum) - if blk != nil { - //consensus.ReshardingNextLeader(blk) - } else { - //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum, "blk is nil") - } } // Collect private keys that are part of the current committee. diff --git a/consensus/view_change.go b/consensus/view_change.go index 3bd0c181f2..6631051dcc 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -372,7 +372,6 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri // onViewChange is called when the view change message is received. func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { - //fmt.Printf("[onViewChange] received view change message from %+v\n", recvMsg) consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -395,13 +394,6 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { return } - consensus.getLogger().Debug(). - Err(err). - Interface("SenderPubkeys", recvMsg.SenderPubkeys). - Str("NextLeader", recvMsg.LeaderPubkey.Bytes.Hex()). - Str("myBLSPubKey", consensus.priKey.GetPublicKeys().SerializeToHexStr()). - Msg("[onViewChange] I am the Leader") - if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) { consensus.getLogger().Info(). Int64("have", consensus.Decider.SignersCount(quorum.ViewChange)). diff --git a/node/node_newblock.go b/node/node_newblock.go index 8738a98076..03fd69d9dd 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -2,7 +2,6 @@ package node import ( "errors" - "fmt" "sort" "strings" "time" @@ -89,8 +88,6 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp newBlock, err := node.ProposeNewBlock(newCommitSigsChan) if err == nil { - fmt.Printf("ProposeNewBlock: #%d :%d e:%d, sh:%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), node.Blockchain().CurrentHeader().Epoch().Uint64(), node.Blockchain().ShardID(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) - if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). Msg("Block with the same number was already proposed, abort.") diff --git a/test/configs/local-resharding.txt b/test/configs/local-resharding.txt index fc3d3e97b4..4ce91ece1c 100644 --- a/test/configs/local-resharding.txt +++ b/test/configs/local-resharding.txt @@ -1,12 +1,25 @@ 127.0.0.1 9000 validator .hmy/65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204.key -127.0.0.1 9002 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key -127.0.0.1 9004 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key -127.0.0.1 9006 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key -127.0.0.1 9008 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key -127.0.0.1 9010 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key -127.0.0.1 9099 explorer null 0 +127.0.0.1 9002 validator .hmy/40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809.key +127.0.0.1 9004 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key +127.0.0.1 9006 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key +127.0.0.1 9008 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key +127.0.0.1 9010 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key +127.0.0.1 9012 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key +127.0.0.1 9014 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key +127.0.0.1 9016 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key +127.0.0.1 9018 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key +127.0.0.1 9020 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key +127.0.0.1 9022 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key +127.0.0.1 9200 explorer null 0 127.0.0.1 9100 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key -127.0.0.1 9102 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key -127.0.0.1 9104 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key -127.0.0.1 9106 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key \ No newline at end of file +127.0.0.1 9102 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key +127.0.0.1 9104 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key +127.0.0.1 9106 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key +127.0.0.1 9108 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key +127.0.0.1 9110 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key +127.0.0.1 9112 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key +127.0.0.1 9114 validator .hmy/f47238daef97d60deedbde5302d05dea5de67608f11f406576e363661f7dcbc4a1385948549b31a6c70f6fde8a391486.key +127.0.0.1 9116 validator .hmy/fc4b9c535ee91f015efff3f32fbb9d32cdd9bfc8a837bb3eee89b8fff653c7af2050a4e147ebe5c7233dc2d5df06ee0a.key +127.0.0.1 9118 validator .hmy/ca86e551ee42adaaa6477322d7db869d3e203c00d7b86c82ebee629ad79cb6d57b8f3db28336778ec2180e56a8e07296.key +127.0.0.1 9300 explorer null 1 \ No newline at end of file diff --git a/test/deploy.sh b/test/deploy.sh index 5a949637e9..6bbb12eb9b 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -69,7 +69,7 @@ function launch_localnet() { if ${VERBOSE}; then verbosity=5 else - verbosity=5 + verbosity=3 fi base_args=(--log_folder "${log_folder}" --min_peers "${MIN}" --bootnodes "${BN_MA}" "--network_type=$NETWORK" --blspass file:"${ROOT}/.hmy/blspass.txt" "--dns=false" "--verbosity=${verbosity}" "--p2p.security.max-conn-per-ip=100") @@ -80,11 +80,8 @@ function launch_localnet() { while IFS='' read -r line || [[ -n "$line" ]]; do i=$((i + 1)) - - # Read config for i-th node form config file IFS=' ' read -r ip port mode bls_key shard node_config <<<"${line}" - echo "LINE: ${line} ${shard}" args=("${base_args[@]}" --ip "${ip}" --port "${port}" --key "/tmp/${ip}-${port}.key" --db_dir "${ROOT}/db-${ip}-${port}" "--broadcast_invalid_tx=false") if [[ -z "$ip" || -z "$port" ]]; then echo "skip empty node" From 998bb642c5622a1ef6a67d7c95d89ffb45c12b4a Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 19 Sep 2022 21:40:21 +0700 Subject: [PATCH 159/420] Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. --- consensus/consensus_v2.go | 14 ++-------- core/blockchain.go | 3 ++ core/blockchain_impl.go | 59 +++++++++++++++++++++++++++++++++++++++ core/blockchain_stub.go | 5 ++++ 4 files changed, 69 insertions(+), 12 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index e87ea21012..5fe176c2f4 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -11,7 +11,6 @@ import ( "github.com/ethereum/go-ethereum/common" bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" - "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" @@ -697,7 +696,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { return } // Check if the same leader. - pub, err := chain.GetLeaderPubKeyFromCoinbase(consensus.Blockchain, header) + pub, err := consensus.Blockchain.GetLeaderPubKeyFromCoinbase(header) if err != nil { utils.Logger().Error().Err(err).Msg("Failed to get leader public key from coinbase") return @@ -708,16 +707,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { } } // Passed all checks, we can change leader. - var ( - wasFound bool - next *bls.PublicKeyWrapper - ) - // The same leader for N blocks. - if consensus.Blockchain.Config().IsAllowlistEpoch(epoch) { - wasFound, next = consensus.Decider.NthNextHmyExt(shard.Schedule.InstanceForEpoch(epoch), leader, 1) - } else { - wasFound, next = consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) - } + wasFound, next := consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) if !wasFound { utils.Logger().Error().Msg("Failed to get next leader") return diff --git a/core/blockchain.go b/core/blockchain.go index 1c7ea43d33..fda4831655 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -13,6 +13,7 @@ import ( "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" + "github.com/harmony-one/harmony/crypto/bls" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/tikv/redis_helper" @@ -334,6 +335,8 @@ type BlockChain interface { state *state.DB, ) (status WriteStatus, err error) + GetLeaderPubKeyFromCoinbase(h *block.Header) (*bls.PublicKeyWrapper, error) + // ========== Only For Tikv Start ========== // return true if is tikv writer master diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 73f37f8623..489fb3661e 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -39,6 +39,7 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" + bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/block" consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/consensus/reward" @@ -47,6 +48,7 @@ import ( "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" + "github.com/harmony-one/harmony/crypto/bls" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/tikv" @@ -181,6 +183,7 @@ type BlockChainImpl struct { validatorListByDelegatorCache *lru.Cache // Cache of validator list by delegator pendingCrossLinksCache *lru.Cache // Cache of last pending crosslinks blockAccumulatorCache *lru.Cache // Cache of block accumulators + leaderPubKeyFromCoinbase *lru.Cache // Cache of leader public key from coinbase quit chan struct{} // blockchain quit channel running int32 // running must be called atomically blockchainPruner *blockchainPruner // use to prune beacon chain @@ -242,6 +245,7 @@ func newBlockChainWithOptions( validatorListByDelegatorCache, _ := lru.New(validatorListByDelegatorCacheLimit) pendingCrossLinksCache, _ := lru.New(pendingCrossLinksCacheLimit) blockAccumulatorCache, _ := lru.New(blockAccumulatorCacheLimit) + leaderPubKeyFromCoinbase, _ := lru.New(chainConfig.LeaderRotationBlocksCount + 2) bc := &BlockChainImpl{ chainConfig: chainConfig, @@ -265,6 +269,7 @@ func newBlockChainWithOptions( validatorListByDelegatorCache: validatorListByDelegatorCache, pendingCrossLinksCache: pendingCrossLinksCache, blockAccumulatorCache: blockAccumulatorCache, + leaderPubKeyFromCoinbase: leaderPubKeyFromCoinbase, blockchainPruner: newBlockchainPruner(db), engine: engine, vmConfig: vmConfig, @@ -3256,6 +3261,60 @@ func (bc *BlockChainImpl) SuperCommitteeForNextEpoch( return nextCommittee, err } +// GetLeaderPubKeyFromCoinbase retrieve corresponding blsPublicKey from Coinbase Address +func (bc *BlockChainImpl) GetLeaderPubKeyFromCoinbase(h *block.Header) (*bls.PublicKeyWrapper, error) { + if cached, ok := bc.leaderPubKeyFromCoinbase.Get(h.Number().Uint64()); ok { + return cached.(*bls.PublicKeyWrapper), nil + } + rs, err := bc.getLeaderPubKeyFromCoinbase(h) + if err != nil { + return nil, err + } + bc.leaderPubKeyFromCoinbase.Add(h.Number().Uint64(), rs) + return rs, nil +} + +// getLeaderPubKeyFromCoinbase retrieve corresponding blsPublicKey from Coinbase Address +func (bc *BlockChainImpl) getLeaderPubKeyFromCoinbase(h *block.Header) (*bls.PublicKeyWrapper, error) { + shardState, err := bc.ReadShardState(h.Epoch()) + if err != nil { + return nil, errors.Wrapf(err, "cannot read shard state %v %s", + h.Epoch(), + h.Coinbase().Hash().Hex(), + ) + } + + committee, err := shardState.FindCommitteeByID(h.ShardID()) + if err != nil { + return nil, err + } + + committerKey := new(bls2.PublicKey) + isStaking := bc.Config().IsStaking(h.Epoch()) + for _, member := range committee.Slots { + if isStaking { + // After staking the coinbase address will be the address of bls public key + if utils.GetAddressFromBLSPubKeyBytes(member.BLSPublicKey[:]) == h.Coinbase() { + if committerKey, err = bls.BytesToBLSPublicKey(member.BLSPublicKey[:]); err != nil { + return nil, err + } + return &bls.PublicKeyWrapper{Object: committerKey, Bytes: member.BLSPublicKey}, nil + } + } else { + if member.EcdsaAddress == h.Coinbase() { + if committerKey, err = bls.BytesToBLSPublicKey(member.BLSPublicKey[:]); err != nil { + return nil, err + } + return &bls.PublicKeyWrapper{Object: committerKey, Bytes: member.BLSPublicKey}, nil + } + } + } + return nil, errors.Errorf( + "cannot find corresponding BLS Public Key coinbase %s", + h.Coinbase().Hex(), + ) +} + func (bc *BlockChainImpl) EnablePruneBeaconChainFeature() { bc.pruneBeaconChainEnable = true } diff --git a/core/blockchain_stub.go b/core/blockchain_stub.go index ccb1c98474..f9e9111eae 100644 --- a/core/blockchain_stub.go +++ b/core/blockchain_stub.go @@ -13,6 +13,7 @@ import ( "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" + "github.com/harmony-one/harmony/crypto/bls" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/tikv/redis_helper" @@ -403,6 +404,10 @@ func (a Stub) CommitOffChainData(batch rawdb.DatabaseWriter, block *types.Block, return 0, errors.Errorf("method CommitOffChainData not implemented for %s", a.Name) } +func (a Stub) GetLeaderPubKeyFromCoinbase(h *block.Header) (*bls.PublicKeyWrapper, error) { + return nil, errors.Errorf("method GetLeaderPubKeyFromCoinbase not implemented for %s", a.Name) +} + func (a Stub) IsTikvWriterMaster() bool { return false } From 77269ef9db28f68b799890ec83e0e916e4d6ad8c Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 17 Oct 2022 15:08:07 +0700 Subject: [PATCH 160/420] activate epoch --- internal/params/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/params/config.go b/internal/params/config.go index 55e8126e80..9fa4a460a5 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -107,6 +107,8 @@ var ( ChainIdFixEpoch: big.NewInt(0), CrossShardXferPrecompileEpoch: big.NewInt(2), AllowlistEpoch: big.NewInt(2), + LeaderRotationEpoch: big.NewInt(290), + LeaderRotationBlocksCount: 64, FeeCollectEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. From 3d1806792f76da55e7851db7f39fca24fe9701af Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 17 Oct 2022 17:09:04 +0700 Subject: [PATCH 161/420] comment activation --- internal/params/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/params/config.go b/internal/params/config.go index 9fa4a460a5..d7b6dd7e29 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -107,8 +107,8 @@ var ( ChainIdFixEpoch: big.NewInt(0), CrossShardXferPrecompileEpoch: big.NewInt(2), AllowlistEpoch: big.NewInt(2), - LeaderRotationEpoch: big.NewInt(290), - LeaderRotationBlocksCount: 64, + //LeaderRotationEpoch: big.NewInt(290), + LeaderRotationBlocksCount: 64, FeeCollectEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. From f2bde96da9e51efc296c67a9ca734ab81d412012 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 18 Oct 2022 12:17:17 +0700 Subject: [PATCH 162/420] 295 epoch --- internal/params/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/params/config.go b/internal/params/config.go index d7b6dd7e29..fd920c3d83 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -107,8 +107,8 @@ var ( ChainIdFixEpoch: big.NewInt(0), CrossShardXferPrecompileEpoch: big.NewInt(2), AllowlistEpoch: big.NewInt(2), - //LeaderRotationEpoch: big.NewInt(290), - LeaderRotationBlocksCount: 64, + LeaderRotationEpoch: big.NewInt(295), + LeaderRotationBlocksCount: 64, FeeCollectEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. From f156543fdcbf102d7038a9079398ed8e8f89debf Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 8 Nov 2022 20:19:37 +0700 Subject: [PATCH 163/420] Fix failed tests. --- internal/params/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/params/config.go b/internal/params/config.go index fd920c3d83..5d968ddef7 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -305,8 +305,8 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } @@ -350,7 +350,7 @@ var ( // TODO place correct epoch number big.NewInt(1), // LeaderRotationEpoch 64, // LeaderRotationBlocksCount - big.NewInt(0), // FeeCollectEpoch + big.NewInt(0), // FeeCollectEpoch } // TestRules ... From 5a52b7e70d220ab44c32e4f0a9f048d0ee312376 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 9 Nov 2022 15:36:28 +0700 Subject: [PATCH 164/420] Fixed code review. --- consensus/consensus_v2.go | 2 +- hmy/hmy.go | 2 -- internal/params/config.go | 11 ++++++----- node/api.go | 6 ------ 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 5fe176c2f4..9d8702f63c 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -691,7 +691,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { if header == nil { return } - // Previous epoch, we should not change leader. + // Previous block was epoch block, we should not change leader. if header.Epoch().Uint64() != epoch.Uint64() { return } diff --git a/hmy/hmy.go b/hmy/hmy.go index 8089f04ba8..58e8d59edf 100644 --- a/hmy/hmy.go +++ b/hmy/hmy.go @@ -18,7 +18,6 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" - "github.com/harmony-one/harmony/multibls" commonRPC "github.com/harmony-one/harmony/rpc/common" "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" @@ -97,7 +96,6 @@ type NodeAPI interface { GetStakingTransactionsCount(address, txType string) (uint64, error) GetTraceResultByHash(hash common.Hash) (json.RawMessage, error) IsCurrentlyLeader() bool - GetPublicKeys() multibls.PublicKeys IsOutOfSync(shardID uint32) bool SyncStatus(shardID uint32) (bool, uint64, uint64) SyncPeers() map[string]int diff --git a/internal/params/config.go b/internal/params/config.go index 5d968ddef7..b69c8bee3e 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -226,6 +226,8 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, FeeCollectEpoch: EpochTBD, + LeaderRotationEpoch: EpochTBD, + LeaderRotationBlocksCount: 64, } // LocalnetChainConfig contains the chain parameters to run for local development. @@ -264,7 +266,7 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, FeeCollectEpoch: big.NewInt(5), - LeaderRotationEpoch: big.NewInt(4), + LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 5, } @@ -347,10 +349,9 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - // TODO place correct epoch number - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount - big.NewInt(0), // FeeCollectEpoch + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount + big.NewInt(0), // FeeCollectEpoch } // TestRules ... diff --git a/node/api.go b/node/api.go index debcb201f6..ceda968084 100644 --- a/node/api.go +++ b/node/api.go @@ -6,7 +6,6 @@ import ( "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy" "github.com/harmony-one/harmony/internal/tikv" - "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/rosetta" hmy_rpc "github.com/harmony-one/harmony/rpc" rpc_common "github.com/harmony-one/harmony/rpc/common" @@ -19,11 +18,6 @@ func (node *Node) IsCurrentlyLeader() bool { return node.Consensus.IsLeader() } -// GetPublicKeys exposes if node is currently the leader node -func (node *Node) GetPublicKeys() multibls.PublicKeys { - return node.Consensus.GetPrivateKeys().GetPublicKeys() -} - // PeerConnectivity .. func (node *Node) PeerConnectivity() (int, int, int) { return node.host.PeerConnectivity() From 14428c96b6c8fa569c8db46a62ea0d56d75cafc1 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 10 Nov 2022 01:42:00 +0700 Subject: [PATCH 165/420] Fix review "--port flag". --- internal/utils/singleton.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index 7aee4d86be..10101d7673 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -197,6 +197,7 @@ func updateZeroLogLevel(level int) { zeroLogger = &childLogger } +// GetPort is useful for debugging, returns `--port` flag provided to executable. func GetPort() int { ok := false for _, x := range os.Args { From 5316d54efaefe7c073b8baa299b813e7b02c1346 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 10 Nov 2022 12:57:57 +0700 Subject: [PATCH 166/420] Fix review comments. --- consensus/consensus.go | 9 ++++++ consensus/consensus_v2.go | 66 +++++++++++++++++++-------------------- internal/params/config.go | 6 ++++ 3 files changed, 48 insertions(+), 33 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 1259576834..e0e096e306 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -180,12 +180,21 @@ func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper { defer consensus.pubKeyLock.Unlock() return consensus.LeaderPubKey } + +func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { + return consensus.LeaderPubKey +} + func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { consensus.pubKeyLock.Lock() consensus.LeaderPubKey = pub consensus.pubKeyLock.Unlock() } +func (consensus *Consensus) setLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { + consensus.LeaderPubKey = pub +} + func (consensus *Consensus) GetPrivateKeys() multibls.PrivateKeys { return consensus.priKey } diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 9d8702f63c..315fc332e7 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -680,46 +680,46 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess return nil } +// rotateLeader rotates the leader to the next leader in the committee. +// This function must be called with enabled leader rotation. func (consensus *Consensus) rotateLeader(epoch *big.Int) { - prev := consensus.GetLeaderPubKey() + prev := consensus.getLeaderPubKey() curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), consensus.Blockchain.Config().IsLeaderRotation(epoch), consensus.Blockchain.Config().LeaderRotationBlocksCount) - if consensus.Blockchain.Config().IsLeaderRotation(epoch) { - leader := consensus.GetLeaderPubKey() - for i := 0; i < consensus.Blockchain.Config().LeaderRotationBlocksCount; i++ { - header := consensus.Blockchain.GetHeaderByNumber(curNumber - uint64(i)) - if header == nil { - return - } - // Previous block was epoch block, we should not change leader. - if header.Epoch().Uint64() != epoch.Uint64() { - return - } - // Check if the same leader. - pub, err := consensus.Blockchain.GetLeaderPubKeyFromCoinbase(header) - if err != nil { - utils.Logger().Error().Err(err).Msg("Failed to get leader public key from coinbase") - return - } - if !pub.Object.IsEqual(leader.Object) { - // Another leader. - return - } + leader := consensus.getLeaderPubKey() + for i := 0; i < consensus.Blockchain.Config().LeaderRotationBlocksCount; i++ { + header := consensus.Blockchain.GetHeaderByNumber(curNumber - uint64(i)) + if header == nil { + return } - // Passed all checks, we can change leader. - wasFound, next := consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) - if !wasFound { - utils.Logger().Error().Msg("Failed to get next leader") + // Previous epoch, we should not change leader. + if header.Epoch().Uint64() != epoch.Uint64() { return - } else { - consensus.SetLeaderPubKey(next) } - if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { - // leader changed - go func() { - consensus.ReadySignal <- SyncProposal - }() + // Check if the same leader. + pub, err := consensus.Blockchain.GetLeaderPubKeyFromCoinbase(header) + if err != nil { + utils.Logger().Error().Err(err).Msg("Failed to get leader public key from coinbase") + return } + if !pub.Object.IsEqual(leader.Object) { + // Another leader. + return + } + } + // Passed all checks, we can change leader. + wasFound, next := consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) + if !wasFound { + utils.Logger().Error().Msg("Failed to get next leader") + return + } else { + consensus.setLeaderPubKey(next) + } + if consensus.isLeader() && !consensus.getLeaderPubKey().Object.IsEqual(prev.Object) { + // leader changed + go func() { + consensus.ReadySignal <- SyncProposal + }() } } diff --git a/internal/params/config.go b/internal/params/config.go index b69c8bee3e..3d40a7c6c9 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -70,6 +70,8 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(1323), // Around Wed 8 Feb 11:30PM UTC AllowlistEpoch: EpochTBD, FeeCollectEpoch: EpochTBD, + LeaderRotationEpoch: EpochTBD, + LeaderRotationBlocksCount: 64, } // TestnetChainConfig contains the chain parameters to run a node on the harmony test network. @@ -147,6 +149,8 @@ var ( SlotsLimitedEpoch: EpochTBD, // epoch to enable HIP-16 CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, + LeaderRotationEpoch: EpochTBD, + LeaderRotationBlocksCount: 64, FeeCollectEpoch: EpochTBD, } @@ -187,6 +191,8 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, FeeCollectEpoch: big.NewInt(574), + LeaderRotationEpoch: EpochTBD, + LeaderRotationBlocksCount: 64, } // StressnetChainConfig contains the chain parameters for the Stress test network. From deda2cb4e246ea862377744c6138a240f764a683 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 10 Nov 2022 17:00:38 +0700 Subject: [PATCH 167/420] Returned locks in rotateLeader. --- consensus/consensus_v2.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 315fc332e7..ae57632481 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -683,10 +683,10 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess // rotateLeader rotates the leader to the next leader in the committee. // This function must be called with enabled leader rotation. func (consensus *Consensus) rotateLeader(epoch *big.Int) { - prev := consensus.getLeaderPubKey() + prev := consensus.GetLeaderPubKey() curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), consensus.Blockchain.Config().IsLeaderRotation(epoch), consensus.Blockchain.Config().LeaderRotationBlocksCount) - leader := consensus.getLeaderPubKey() + leader := consensus.GetLeaderPubKey() for i := 0; i < consensus.Blockchain.Config().LeaderRotationBlocksCount; i++ { header := consensus.Blockchain.GetHeaderByNumber(curNumber - uint64(i)) if header == nil { @@ -713,9 +713,9 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { utils.Logger().Error().Msg("Failed to get next leader") return } else { - consensus.setLeaderPubKey(next) + consensus.SetLeaderPubKey(next) } - if consensus.isLeader() && !consensus.getLeaderPubKey().Object.IsEqual(prev.Object) { + if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { // leader changed go func() { consensus.ReadySignal <- SyncProposal From dbce3f5fe2e4f8adc8f5661aa83d684cc56cd221 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 17 Jan 2023 11:57:05 -0300 Subject: [PATCH 168/420] Rebased onto dev. --- consensus/consensus_v2.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index ae57632481..0b1ce28cf4 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -684,11 +684,12 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess // This function must be called with enabled leader rotation. func (consensus *Consensus) rotateLeader(epoch *big.Int) { prev := consensus.GetLeaderPubKey() - curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() - utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), consensus.Blockchain.Config().IsLeaderRotation(epoch), consensus.Blockchain.Config().LeaderRotationBlocksCount) + bc := consensus.Blockchain() + curNumber := bc.CurrentHeader().Number().Uint64() + utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), bc.Config().IsLeaderRotation(epoch), bc.Config().LeaderRotationBlocksCount) leader := consensus.GetLeaderPubKey() - for i := 0; i < consensus.Blockchain.Config().LeaderRotationBlocksCount; i++ { - header := consensus.Blockchain.GetHeaderByNumber(curNumber - uint64(i)) + for i := 0; i < bc.Config().LeaderRotationBlocksCount; i++ { + header := bc.GetHeaderByNumber(curNumber - uint64(i)) if header == nil { return } @@ -697,7 +698,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { return } // Check if the same leader. - pub, err := consensus.Blockchain.GetLeaderPubKeyFromCoinbase(header) + pub, err := bc.GetLeaderPubKeyFromCoinbase(header) if err != nil { utils.Logger().Error().Err(err).Msg("Failed to get leader public key from coinbase") return @@ -736,7 +737,7 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg } else { epoch = blk.Epoch() } - if consensus.Blockchain.Config().IsLeaderRotation(epoch) { + if consensus.Blockchain().Config().IsLeaderRotation(epoch) { consensus.rotateLeader(epoch) } From b242506df550ade4fa10911c0a8303598002fb9b Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 17 Jan 2023 17:17:30 -0300 Subject: [PATCH 169/420] Commented golangci. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9d4982491b..514f8e2fb4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ install: - (cd $GOPATH/src/github.com/harmony-one/bls; make BLS_SWAP_G=1 -j4) - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1 - - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.41.1 +# - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.41.1 - make go-get - go install golang.org/x/tools/cmd/goimports@latest - go install github.com/harmony-ek/gencodec@latest From dc3ee197f95275c48bcc0446b080a57530845741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <“Gheis.Mohammadi@gmail.com”> Date: Tue, 27 Dec 2022 16:49:02 +0800 Subject: [PATCH 170/420] staged stream sync v1.0 --- .gitignore | 3 + api/service/manager.go | 3 + api/service/stagedstreamsync/adapter.go | 34 + api/service/stagedstreamsync/beacon_helper.go | 156 +++++ .../stagedstreamsync/block_by_hash_manager.go | 133 ++++ .../stagedstreamsync/block_hash_result.go | 75 +++ api/service/stagedstreamsync/block_manager.go | 172 +++++ api/service/stagedstreamsync/const.go | 81 +++ .../stagedstreamsync/default_stages.go | 87 +++ api/service/stagedstreamsync/downloader.go | 303 +++++++++ api/service/stagedstreamsync/downloaders.go | 96 +++ api/service/stagedstreamsync/errors.go | 56 ++ api/service/stagedstreamsync/helpers.go | 114 ++++ api/service/stagedstreamsync/metric.go | 98 +++ api/service/stagedstreamsync/service.go | 30 + .../stagedstreamsync/short_range_helper.go | 221 +++++++ api/service/stagedstreamsync/stage.go | 112 ++++ api/service/stagedstreamsync/stage_bodies.go | 420 ++++++++++++ api/service/stagedstreamsync/stage_epoch.go | 198 ++++++ api/service/stagedstreamsync/stage_finish.go | 114 ++++ api/service/stagedstreamsync/stage_heads.go | 157 +++++ .../stagedstreamsync/stage_short_range.go | 205 ++++++ api/service/stagedstreamsync/stage_state.go | 295 +++++++++ .../stagedstreamsync/staged_stream_sync.go | 597 ++++++++++++++++++ api/service/stagedstreamsync/stages.go | 71 +++ api/service/stagedstreamsync/syncing.go | 320 ++++++++++ api/service/stagedstreamsync/types.go | 287 +++++++++ api/service/stagedstreamsync/types_test.go | 266 ++++++++ cmd/harmony/main.go | 46 +- go.mod | 1 + go.sum | 2 + hmy/downloader/const.go | 3 +- hmy/downloader/metric.go | 14 +- hmy/downloader/shortrange.go | 2 +- node/node_syncing.go | 44 +- .../common/requestmanager/interface_test.go | 12 + .../common/streammanager/interface_test.go | 12 + .../common/streammanager/streammanager.go | 23 +- p2p/stream/protocols/sync/client.go | 30 + p2p/stream/protocols/sync/const.go | 3 + p2p/stream/protocols/sync/protocol.go | 62 +- p2p/stream/types/stream.go | 22 +- p2p/stream/types/utils.go | 37 +- 43 files changed, 4968 insertions(+), 49 deletions(-) create mode 100644 api/service/stagedstreamsync/adapter.go create mode 100644 api/service/stagedstreamsync/beacon_helper.go create mode 100644 api/service/stagedstreamsync/block_by_hash_manager.go create mode 100644 api/service/stagedstreamsync/block_hash_result.go create mode 100644 api/service/stagedstreamsync/block_manager.go create mode 100644 api/service/stagedstreamsync/const.go create mode 100644 api/service/stagedstreamsync/default_stages.go create mode 100644 api/service/stagedstreamsync/downloader.go create mode 100644 api/service/stagedstreamsync/downloaders.go create mode 100644 api/service/stagedstreamsync/errors.go create mode 100644 api/service/stagedstreamsync/helpers.go create mode 100644 api/service/stagedstreamsync/metric.go create mode 100644 api/service/stagedstreamsync/service.go create mode 100644 api/service/stagedstreamsync/short_range_helper.go create mode 100644 api/service/stagedstreamsync/stage.go create mode 100644 api/service/stagedstreamsync/stage_bodies.go create mode 100644 api/service/stagedstreamsync/stage_epoch.go create mode 100644 api/service/stagedstreamsync/stage_finish.go create mode 100644 api/service/stagedstreamsync/stage_heads.go create mode 100644 api/service/stagedstreamsync/stage_short_range.go create mode 100644 api/service/stagedstreamsync/stage_state.go create mode 100644 api/service/stagedstreamsync/staged_stream_sync.go create mode 100644 api/service/stagedstreamsync/stages.go create mode 100644 api/service/stagedstreamsync/syncing.go create mode 100644 api/service/stagedstreamsync/types.go create mode 100644 api/service/stagedstreamsync/types_test.go diff --git a/.gitignore b/.gitignore index 0e26bf59ae..6a3b3c4267 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,6 @@ explorer_storage_* # pprof profiles profiles/*.pb.gz + +# cache db +cache_*_db \ No newline at end of file diff --git a/api/service/manager.go b/api/service/manager.go index 57ca15d607..d879b6a6cd 100644 --- a/api/service/manager.go +++ b/api/service/manager.go @@ -23,6 +23,7 @@ const ( Prometheus Synchronize CrosslinkSending + StagedStreamSync ) func (t Type) String() string { @@ -45,6 +46,8 @@ func (t Type) String() string { return "Synchronize" case CrosslinkSending: return "CrosslinkSending" + case StagedStreamSync: + return "StagedStreamSync" default: return "Unknown" } diff --git a/api/service/stagedstreamsync/adapter.go b/api/service/stagedstreamsync/adapter.go new file mode 100644 index 0000000000..ae7632889c --- /dev/null +++ b/api/service/stagedstreamsync/adapter.go @@ -0,0 +1,34 @@ +package stagedstreamsync + +import ( + "context" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/event" + "github.com/harmony-one/harmony/consensus/engine" + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/p2p/stream/common/streammanager" + syncproto "github.com/harmony-one/harmony/p2p/stream/protocols/sync" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" +) + +type syncProtocol interface { + GetCurrentBlockNumber(ctx context.Context, opts ...syncproto.Option) (uint64, sttypes.StreamID, error) + GetBlocksByNumber(ctx context.Context, bns []uint64, opts ...syncproto.Option) ([]*types.Block, sttypes.StreamID, error) + GetRawBlocksByNumber(ctx context.Context, bns []uint64, opts ...syncproto.Option) ([][]byte, [][]byte, sttypes.StreamID, error) + GetBlockHashes(ctx context.Context, bns []uint64, opts ...syncproto.Option) ([]common.Hash, sttypes.StreamID, error) + GetBlocksByHashes(ctx context.Context, hs []common.Hash, opts ...syncproto.Option) ([]*types.Block, sttypes.StreamID, error) + + RemoveStream(stID sttypes.StreamID) // If a stream delivers invalid data, remove the stream + StreamFailed(stID sttypes.StreamID, reason string) + SubscribeAddStreamEvent(ch chan<- streammanager.EvtStreamAdded) event.Subscription + NumStreams() int +} + +type blockChain interface { + engine.ChainReader + Engine() engine.Engine + + InsertChain(chain types.Blocks, verifyHeaders bool) (int, error) + WriteCommitSig(blockNum uint64, lastCommits []byte) error +} diff --git a/api/service/stagedstreamsync/beacon_helper.go b/api/service/stagedstreamsync/beacon_helper.go new file mode 100644 index 0000000000..4a77020160 --- /dev/null +++ b/api/service/stagedstreamsync/beacon_helper.go @@ -0,0 +1,156 @@ +package stagedstreamsync + +import ( + "time" + + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/internal/utils" + "github.com/rs/zerolog" +) + +// lastMileCache keeps the last 50 number blocks in memory cache +const lastMileCap = 50 + +type ( + // beaconHelper is the helper for the beacon downloader. The beaconHelper is only started + // when node is running on side chain, listening to beacon client pub-sub message and + // insert the latest blocks to the beacon chain. + beaconHelper struct { + bc blockChain + blockC <-chan *types.Block + // TODO: refactor this hook to consensus module. We'd better put it in + // consensus module under a subscription. + insertHook func() + + lastMileCache *blocksByNumber + insertC chan insertTask + closeC chan struct{} + logger zerolog.Logger + } + + insertTask struct { + doneC chan struct{} + } +) + +func newBeaconHelper(bc blockChain, blockC <-chan *types.Block, insertHook func()) *beaconHelper { + return &beaconHelper{ + bc: bc, + blockC: blockC, + insertHook: insertHook, + lastMileCache: newBlocksByNumber(lastMileCap), + insertC: make(chan insertTask, 1), + closeC: make(chan struct{}), + logger: utils.Logger().With(). + Str("module", "downloader"). + Str("sub-module", "beacon helper"). + Logger(), + } +} + +func (bh *beaconHelper) start() { + go bh.loop() +} + +func (bh *beaconHelper) close() { + close(bh.closeC) +} + +func (bh *beaconHelper) loop() { + t := time.NewTicker(10 * time.Second) + defer t.Stop() + for { + select { + case <-t.C: + bh.insertAsync() + + case b, ok := <-bh.blockC: + if !ok { + return // blockC closed. Node exited + } + if b == nil { + continue + } + bh.lastMileCache.push(b) + bh.insertAsync() + + case it := <-bh.insertC: + inserted, bn, err := bh.insertLastMileBlocks() + numBlocksInsertedBeaconHelperCounter.Add(float64(inserted)) + if err != nil { + bh.logger.Error().Err(err).Msg(WrapStagedSyncMsg("insert last mile blocks error")) + continue + } + bh.logger.Info().Int("inserted", inserted). + Uint64("end height", bn). + Uint32("shard", bh.bc.ShardID()). + Msg(WrapStagedSyncMsg("insert last mile blocks")) + + close(it.doneC) + + case <-bh.closeC: + return + } + } +} + +// insertAsync triggers the insert last mile without blocking +func (bh *beaconHelper) insertAsync() { + select { + case bh.insertC <- insertTask{ + doneC: make(chan struct{}), + }: + default: + } +} + +// insertSync triggers the insert last mile while blocking +func (bh *beaconHelper) insertSync() { + task := insertTask{ + doneC: make(chan struct{}), + } + bh.insertC <- task + <-task.doneC +} + +func (bh *beaconHelper) insertLastMileBlocks() (inserted int, bn uint64, err error) { + bn = bh.bc.CurrentBlock().NumberU64() + 1 + for { + b := bh.getNextBlock(bn) + if b == nil { + bn-- + return + } + // TODO: Instruct the beacon helper to verify signatures. This may require some forks + // in pub-sub message (add commit sigs in node.block.sync messages) + if _, err = bh.bc.InsertChain(types.Blocks{b}, true); err != nil { + bn-- + return + } + bh.logger.Info().Uint64("number", b.NumberU64()).Msg(WrapStagedSyncMsg("Inserted block from beacon pub-sub")) + + if bh.insertHook != nil { + bh.insertHook() + } + inserted++ + bn++ + } +} + +func (bh *beaconHelper) getNextBlock(expBN uint64) *types.Block { + for bh.lastMileCache.len() > 0 { + b := bh.lastMileCache.pop() + if b == nil { + return nil + } + if b.NumberU64() < expBN { + continue + } + if b.NumberU64() > expBN { + bh.lastMileCache.push(b) + return nil + } + return b + } + return nil +} diff --git a/api/service/stagedstreamsync/block_by_hash_manager.go b/api/service/stagedstreamsync/block_by_hash_manager.go new file mode 100644 index 0000000000..86abc22ed7 --- /dev/null +++ b/api/service/stagedstreamsync/block_by_hash_manager.go @@ -0,0 +1,133 @@ +package stagedstreamsync + +import ( + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/harmony/core/types" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/pkg/errors" +) + +type getBlocksByHashManager struct { + hashes []common.Hash + pendings map[common.Hash]struct{} + results map[common.Hash]blockResult + whitelist []sttypes.StreamID + + lock sync.Mutex +} + +func newGetBlocksByHashManager(hashes []common.Hash, whitelist []sttypes.StreamID) *getBlocksByHashManager { + return &getBlocksByHashManager{ + hashes: hashes, + pendings: make(map[common.Hash]struct{}), + results: make(map[common.Hash]blockResult), + whitelist: whitelist, + } +} + +func (m *getBlocksByHashManager) getNextHashes() ([]common.Hash, []sttypes.StreamID, error) { + m.lock.Lock() + defer m.lock.Unlock() + + num := m.numBlocksPerRequest() + hashes := make([]common.Hash, 0, num) + if len(m.whitelist) == 0 { + return nil, nil, errors.New("empty white list") + } + + for _, hash := range m.hashes { + if len(hashes) == num { + break + } + _, ok1 := m.pendings[hash] + _, ok2 := m.results[hash] + if !ok1 && !ok2 { + hashes = append(hashes, hash) + } + } + sts := make([]sttypes.StreamID, len(m.whitelist)) + copy(sts, m.whitelist) + return hashes, sts, nil +} + +func (m *getBlocksByHashManager) numBlocksPerRequest() int { + val := divideCeil(len(m.hashes), len(m.whitelist)) + if val < BlockByHashesLowerCap { + val = BlockByHashesLowerCap + } + if val > BlockByHashesUpperCap { + val = BlockByHashesUpperCap + } + return val +} + +func (m *getBlocksByHashManager) numRequests() int { + return divideCeil(len(m.hashes), m.numBlocksPerRequest()) +} + +func (m *getBlocksByHashManager) addResult(hashes []common.Hash, blocks []*types.Block, stid sttypes.StreamID) { + m.lock.Lock() + defer m.lock.Unlock() + + for i, hash := range hashes { + block := blocks[i] + delete(m.pendings, hash) + m.results[hash] = blockResult{ + block: block, + stid: stid, + } + } +} + +func (m *getBlocksByHashManager) handleResultError(hashes []common.Hash, stid sttypes.StreamID) { + m.lock.Lock() + defer m.lock.Unlock() + + m.removeStreamID(stid) + + for _, hash := range hashes { + delete(m.pendings, hash) + } +} + +func (m *getBlocksByHashManager) getResults() ([]*types.Block, []sttypes.StreamID, error) { + m.lock.Lock() + defer m.lock.Unlock() + + blocks := make([]*types.Block, 0, len(m.hashes)) + stids := make([]sttypes.StreamID, 0, len(m.hashes)) + for _, hash := range m.hashes { + if m.results[hash].block == nil { + return nil, nil, errors.New("SANITY: nil block found") + } + blocks = append(blocks, m.results[hash].block) + stids = append(stids, m.results[hash].stid) + } + return blocks, stids, nil +} + +func (m *getBlocksByHashManager) isDone() bool { + m.lock.Lock() + defer m.lock.Unlock() + + return len(m.results) == len(m.hashes) +} + +func (m *getBlocksByHashManager) removeStreamID(target sttypes.StreamID) { + // O(n^2) complexity. But considering the whitelist size is small, should not + // have performance issue. +loop: + for i, stid := range m.whitelist { + if stid == target { + if i == len(m.whitelist) { + m.whitelist = m.whitelist[:i] + } else { + m.whitelist = append(m.whitelist[:i], m.whitelist[i+1:]...) + } + goto loop + } + } + return +} diff --git a/api/service/stagedstreamsync/block_hash_result.go b/api/service/stagedstreamsync/block_hash_result.go new file mode 100644 index 0000000000..0bae60507d --- /dev/null +++ b/api/service/stagedstreamsync/block_hash_result.go @@ -0,0 +1,75 @@ +package stagedstreamsync + +import ( + "sync" + + "github.com/ethereum/go-ethereum/common" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" +) + +type ( + blockHashResults struct { + bns []uint64 + results []map[sttypes.StreamID]common.Hash + + lock sync.Mutex + } +) + +func newBlockHashResults(bns []uint64) *blockHashResults { + results := make([]map[sttypes.StreamID]common.Hash, 0, len(bns)) + for range bns { + results = append(results, make(map[sttypes.StreamID]common.Hash)) + } + return &blockHashResults{ + bns: bns, + results: results, + } +} + +func (res *blockHashResults) addResult(hashes []common.Hash, stid sttypes.StreamID) { + res.lock.Lock() + defer res.lock.Unlock() + + for i, h := range hashes { + if h == emptyHash { + return // nil block hash reached + } + res.results[i][stid] = h + } + return +} + +func (res *blockHashResults) computeLongestHashChain() ([]common.Hash, []sttypes.StreamID) { + var ( + whitelist map[sttypes.StreamID]struct{} + hashChain []common.Hash + ) + for _, result := range res.results { + hash, nextWl := countHashMaxVote(result, whitelist) + if hash == emptyHash { + break + } + hashChain = append(hashChain, hash) + whitelist = nextWl + } + + sts := make([]sttypes.StreamID, 0, len(whitelist)) + for st := range whitelist { + sts = append(sts, st) + } + return hashChain, sts +} + +func (res *blockHashResults) numBlocksWithResults() int { + res.lock.Lock() + defer res.lock.Unlock() + + cnt := 0 + for _, result := range res.results { + if len(result) != 0 { + cnt++ + } + } + return cnt +} diff --git a/api/service/stagedstreamsync/block_manager.go b/api/service/stagedstreamsync/block_manager.go new file mode 100644 index 0000000000..df30ab5e36 --- /dev/null +++ b/api/service/stagedstreamsync/block_manager.go @@ -0,0 +1,172 @@ +package stagedstreamsync + +import ( + "sync" + + sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/rs/zerolog" +) + +type BlockDownloadDetails struct { + loopID int + streamID sttypes.StreamID +} + +// blockDownloadManager is the helper structure for get blocks request management +type blockDownloadManager struct { + chain blockChain + tx kv.RwTx + + targetBN uint64 + requesting map[uint64]struct{} // block numbers that have been assigned to workers but not received + processing map[uint64]struct{} // block numbers received requests but not inserted + retries *prioritizedNumbers // requests where error happens + rq *resultQueue // result queue wait to be inserted into blockchain + bdd map[uint64]BlockDownloadDetails // details about how this block was downloaded + + logger zerolog.Logger + lock sync.Mutex +} + +func newBlockDownloadManager(tx kv.RwTx, chain blockChain, targetBN uint64, logger zerolog.Logger) *blockDownloadManager { + return &blockDownloadManager{ + chain: chain, + tx: tx, + targetBN: targetBN, + requesting: make(map[uint64]struct{}), + processing: make(map[uint64]struct{}), + retries: newPrioritizedNumbers(), + rq: newResultQueue(), + bdd: make(map[uint64]BlockDownloadDetails), + logger: logger, + } +} + +// GetNextBatch get the next block numbers batch +func (gbm *blockDownloadManager) GetNextBatch() []uint64 { + gbm.lock.Lock() + defer gbm.lock.Unlock() + + cap := BlocksPerRequest + + bns := gbm.getBatchFromRetries(cap) + if len(bns) > 0 { + cap -= len(bns) + gbm.addBatchToRequesting(bns) + } + + if gbm.availableForMoreTasks() { + addBNs := gbm.getBatchFromUnprocessed(cap) + gbm.addBatchToRequesting(addBNs) + bns = append(bns, addBNs...) + } + + return bns +} + +// HandleRequestError handles the error result +func (gbm *blockDownloadManager) HandleRequestError(bns []uint64, err error, streamID sttypes.StreamID) { + gbm.lock.Lock() + defer gbm.lock.Unlock() + + // add requested block numbers to retries + for _, bn := range bns { + delete(gbm.requesting, bn) + gbm.retries.push(bn) + } +} + +// HandleRequestResult handles get blocks result +func (gbm *blockDownloadManager) HandleRequestResult(bns []uint64, blockBytes [][]byte, sigBytes [][]byte, loopID int, streamID sttypes.StreamID) error { + gbm.lock.Lock() + defer gbm.lock.Unlock() + + for i, bn := range bns { + delete(gbm.requesting, bn) + if len(blockBytes[i]) <= 1 { + gbm.retries.push(bn) + } else { + gbm.processing[bn] = struct{}{} + gbm.bdd[bn] = BlockDownloadDetails{ + loopID: loopID, + streamID: streamID, + } + } + } + return nil +} + +// SetDownloadDetails sets the download details for a batch of blocks +func (gbm *blockDownloadManager) SetDownloadDetails(bns []uint64, loopID int, streamID sttypes.StreamID) error { + gbm.lock.Lock() + defer gbm.lock.Unlock() + + for _, bn := range bns { + gbm.bdd[bn] = BlockDownloadDetails{ + loopID: loopID, + streamID: streamID, + } + } + return nil +} + +// GetDownloadDetails returns the download details for a block +func (gbm *blockDownloadManager) GetDownloadDetails(blockNumber uint64) (loopID int, streamID sttypes.StreamID) { + gbm.lock.Lock() + defer gbm.lock.Unlock() + + return gbm.bdd[blockNumber].loopID, gbm.bdd[blockNumber].streamID +} + +// getBatchFromRetries get the block number batch to be requested from retries. +func (gbm *blockDownloadManager) getBatchFromRetries(cap int) []uint64 { + var ( + requestBNs []uint64 + curHeight = gbm.chain.CurrentBlock().NumberU64() + ) + for cnt := 0; cnt < cap; cnt++ { + bn := gbm.retries.pop() + if bn == 0 { + break // no more retries + } + if bn <= curHeight { + continue + } + requestBNs = append(requestBNs, bn) + } + return requestBNs +} + +// getBatchFromUnprocessed returns a batch of block numbers to be requested from unprocessed. +func (gbm *blockDownloadManager) getBatchFromUnprocessed(cap int) []uint64 { + var ( + requestBNs []uint64 + curHeight = gbm.chain.CurrentBlock().NumberU64() + ) + bn := curHeight + 1 + // TODO: this algorithm can be potentially optimized. + for cnt := 0; cnt < cap && bn <= gbm.targetBN; cnt++ { + for bn <= gbm.targetBN { + _, ok1 := gbm.requesting[bn] + _, ok2 := gbm.processing[bn] + if !ok1 && !ok2 { + requestBNs = append(requestBNs, bn) + bn++ + break + } + bn++ + } + } + return requestBNs +} + +func (gbm *blockDownloadManager) availableForMoreTasks() bool { + return gbm.rq.results.Len() < SoftQueueCap +} + +func (gbm *blockDownloadManager) addBatchToRequesting(bns []uint64) { + for _, bn := range bns { + gbm.requesting[bn] = struct{}{} + } +} diff --git a/api/service/stagedstreamsync/const.go b/api/service/stagedstreamsync/const.go new file mode 100644 index 0000000000..721b44d9f5 --- /dev/null +++ b/api/service/stagedstreamsync/const.go @@ -0,0 +1,81 @@ +package stagedstreamsync + +import ( + "time" + + "github.com/harmony-one/harmony/core/types" + nodeconfig "github.com/harmony-one/harmony/internal/configs/node" +) + +const ( + BlocksPerRequest int = 10 // number of blocks for each request + BlocksPerInsertion int = 50 // number of blocks for each insert batch + BlockHashesPerRequest int = 20 // number of get block hashes for short range sync + BlockByHashesUpperCap int = 10 // number of get blocks by hashes upper cap + BlockByHashesLowerCap int = 3 // number of get blocks by hashes lower cap + + LastMileBlocksThreshold int = 10 + + // SoftQueueCap is the soft cap of size in resultQueue. When the queue size is larger than this limit, + // no more request will be assigned to workers to wait for InsertChain to finish. + SoftQueueCap int = 100 + + // DefaultConcurrency is the default settings for concurrency + DefaultConcurrency = 4 + + // ShortRangeTimeout is the timeout for each short range sync, which allow short range sync + // to restart automatically when stuck in `getBlockHashes` + ShortRangeTimeout = 1 * time.Minute +) + +type ( + // Config is the downloader config + Config struct { + // Only run stream sync protocol as a server. + // TODO: remove this when stream sync is fully up. + ServerOnly bool + + // parameters + Network nodeconfig.NetworkType + Concurrency int // Number of concurrent sync requests + MinStreams int // Minimum number of streams to do sync + InitStreams int // Number of streams requirement for initial bootstrap + + // stream manager config + SmSoftLowCap int + SmHardLowCap int + SmHiCap int + SmDiscBatch int + + // config for beacon config + BHConfig *BeaconHelperConfig + + // log the stage progress + LogProgress bool + } + + // BeaconHelperConfig is the extra config used for beaconHelper which uses + // pub-sub block message to do sync. + BeaconHelperConfig struct { + BlockC <-chan *types.Block + InsertHook func() + } +) + +func (c *Config) fixValues() { + if c.Concurrency == 0 { + c.Concurrency = DefaultConcurrency + } + if c.Concurrency > c.MinStreams { + c.MinStreams = c.Concurrency + } + if c.MinStreams > c.InitStreams { + c.InitStreams = c.MinStreams + } + if c.MinStreams > c.SmSoftLowCap { + c.SmSoftLowCap = c.MinStreams + } + if c.MinStreams > c.SmHardLowCap { + c.SmHardLowCap = c.MinStreams + } +} diff --git a/api/service/stagedstreamsync/default_stages.go b/api/service/stagedstreamsync/default_stages.go new file mode 100644 index 0000000000..6e4808738f --- /dev/null +++ b/api/service/stagedstreamsync/default_stages.go @@ -0,0 +1,87 @@ +package stagedstreamsync + +import ( + "context" +) + +type ForwardOrder []SyncStageID +type RevertOrder []SyncStageID +type CleanUpOrder []SyncStageID + +var DefaultForwardOrder = ForwardOrder{ + Heads, + SyncEpoch, + ShortRange, + BlockBodies, + // Stages below don't use Internet + States, + Finish, +} + +var DefaultRevertOrder = RevertOrder{ + Finish, + States, + BlockBodies, + ShortRange, + SyncEpoch, + Heads, +} + +var DefaultCleanUpOrder = CleanUpOrder{ + Finish, + States, + BlockBodies, + ShortRange, + SyncEpoch, + Heads, +} + +func DefaultStages(ctx context.Context, + headsCfg StageHeadsCfg, + seCfg StageEpochCfg, + srCfg StageShortRangeCfg, + bodiesCfg StageBodiesCfg, + statesCfg StageStatesCfg, + finishCfg StageFinishCfg, +) []*Stage { + + handlerStageHeads := NewStageHeads(headsCfg) + handlerStageShortRange := NewStageShortRange(srCfg) + handlerStageEpochSync := NewStageEpoch(seCfg) + handlerStageBodies := NewStageBodies(bodiesCfg) + handlerStageStates := NewStageStates(statesCfg) + handlerStageFinish := NewStageFinish(finishCfg) + + return []*Stage{ + { + ID: Heads, + Description: "Retrieve Chain Heads", + Handler: handlerStageHeads, + }, + { + ID: SyncEpoch, + Description: "Sync only Last Block of Epoch", + Handler: handlerStageEpochSync, + }, + { + ID: ShortRange, + Description: "Short Range Sync", + Handler: handlerStageShortRange, + }, + { + ID: BlockBodies, + Description: "Retrieve Block Bodies", + Handler: handlerStageBodies, + }, + { + ID: States, + Description: "Update Blockchain State", + Handler: handlerStageStates, + }, + { + ID: Finish, + Description: "Finalize Changes", + Handler: handlerStageFinish, + }, + } +} diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go new file mode 100644 index 0000000000..9fdd2f78f4 --- /dev/null +++ b/api/service/stagedstreamsync/downloader.go @@ -0,0 +1,303 @@ +package stagedstreamsync + +import ( + "context" + "fmt" + "time" + + "github.com/ethereum/go-ethereum/event" + "github.com/pkg/errors" + "github.com/rs/zerolog" + + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/crypto/bls" + "github.com/harmony-one/harmony/internal/chain" + nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/p2p" + "github.com/harmony-one/harmony/p2p/stream/common/streammanager" + "github.com/harmony-one/harmony/p2p/stream/protocols/sync" + "github.com/harmony-one/harmony/shard" +) + +type ( + // Downloader is responsible for sync task of one shard + Downloader struct { + bc blockChain + syncProtocol syncProtocol + bh *beaconHelper + stagedSyncInstance *StagedStreamSync + + downloadC chan struct{} + closeC chan struct{} + ctx context.Context + cancel func() + + config Config + logger zerolog.Logger + } +) + +// NewDownloader creates a new downloader +func NewDownloader(host p2p.Host, bc core.BlockChain, config Config) *Downloader { + config.fixValues() + + sp := sync.NewProtocol(sync.Config{ + Chain: bc, + Host: host.GetP2PHost(), + Discovery: host.GetDiscovery(), + ShardID: nodeconfig.ShardID(bc.ShardID()), + Network: config.Network, + + SmSoftLowCap: config.SmSoftLowCap, + SmHardLowCap: config.SmHardLowCap, + SmHiCap: config.SmHiCap, + DiscBatch: config.SmDiscBatch, + }) + + host.AddStreamProtocol(sp) + + var bh *beaconHelper + if config.BHConfig != nil && bc.ShardID() == shard.BeaconChainShardID { + bh = newBeaconHelper(bc, config.BHConfig.BlockC, config.BHConfig.InsertHook) + } + + logger := utils.Logger().With().Str("module", "StagedStreamSync").Uint32("ShardID", bc.ShardID()).Logger() + + ctx, cancel := context.WithCancel(context.Background()) + + //TODO: use mem db should be in config file + stagedSyncInstance, err := CreateStagedSync(ctx, bc, false, sp, config, logger, config.LogProgress) + if err != nil { + return nil + } + + return &Downloader{ + bc: bc, + syncProtocol: sp, + bh: bh, + stagedSyncInstance: stagedSyncInstance, + + downloadC: make(chan struct{}), + closeC: make(chan struct{}), + ctx: ctx, + cancel: cancel, + + config: config, + logger: logger, + } +} + +// Start starts the downloader +func (d *Downloader) Start() { + go func() { + d.waitForBootFinish() + fmt.Printf("boot completed for shard %d, %d streams are connected\n", d.bc.ShardID(), d.syncProtocol.NumStreams()) + d.loop() + }() + + if d.bh != nil { + d.bh.start() + } +} + +// Close closes the downloader +func (d *Downloader) Close() { + close(d.closeC) + d.cancel() + + if d.bh != nil { + d.bh.close() + } +} + +// DownloadAsync triggers the download async. +func (d *Downloader) DownloadAsync() { + select { + case d.downloadC <- struct{}{}: + consensusTriggeredDownloadCounterVec.With(d.promLabels()).Inc() + + case <-time.After(100 * time.Millisecond): + } +} + +// NumPeers returns the number of peers connected of a specific shard. +func (d *Downloader) NumPeers() int { + return d.syncProtocol.NumStreams() +} + +// IsSyncing returns the current sync status +func (d *Downloader) SyncStatus() (bool, uint64, uint64) { + syncing, target := d.stagedSyncInstance.status.get() + if !syncing { + target = d.bc.CurrentBlock().NumberU64() + } + return syncing, target, 0 +} + +// SubscribeDownloadStarted subscribes download started +func (d *Downloader) SubscribeDownloadStarted(ch chan struct{}) event.Subscription { + d.stagedSyncInstance.evtDownloadStartedSubscribed = true + return d.stagedSyncInstance.evtDownloadStarted.Subscribe(ch) +} + +// SubscribeDownloadFinished subscribes the download finished +func (d *Downloader) SubscribeDownloadFinished(ch chan struct{}) event.Subscription { + d.stagedSyncInstance.evtDownloadFinishedSubscribed = true + return d.stagedSyncInstance.evtDownloadFinished.Subscribe(ch) +} + +// waitForBootFinish waits for stream manager to finish the initial discovery and have +// enough peers to start downloader +func (d *Downloader) waitForBootFinish() { + evtCh := make(chan streammanager.EvtStreamAdded, 1) + sub := d.syncProtocol.SubscribeAddStreamEvent(evtCh) + defer sub.Unsubscribe() + + checkCh := make(chan struct{}, 1) + trigger := func() { + select { + case checkCh <- struct{}{}: + default: + } + } + trigger() + + t := time.NewTicker(10 * time.Second) + defer t.Stop() + for { + select { + case <-t.C: + trigger() + + case <-evtCh: + trigger() + + case <-checkCh: + if d.syncProtocol.NumStreams() >= d.config.InitStreams { + return + } + case <-d.closeC: + return + } + } +} + +func (d *Downloader) loop() { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + initSync := d.bc.ShardID() != shard.BeaconChainShardID + trigger := func() { + select { + case d.downloadC <- struct{}{}: + case <-time.After(100 * time.Millisecond): + } + } + go trigger() + + for { + select { + case <-ticker.C: + go trigger() + + case <-d.downloadC: + addedBN, err := d.stagedSyncInstance.doSync(d.ctx, initSync) + if err != nil { + //TODO: if there is a bad block which can't be resolved + if d.stagedSyncInstance.invalidBlock.Active { + numTriedStreams := len(d.stagedSyncInstance.invalidBlock.StreamID) + // if many streams couldn't solve it, then that's an unresolvable bad block + if numTriedStreams >= d.config.InitStreams { + if !d.stagedSyncInstance.invalidBlock.IsLogged { + fmt.Println("unresolvable bad block:", d.stagedSyncInstance.invalidBlock.Number) + d.stagedSyncInstance.invalidBlock.IsLogged = true + } + //TODO: if we don't have any new or untried stream in the list, sleep or panic + } + } + + // If error happens, sleep 5 seconds and retry + d.logger.Error(). + Err(err). + Bool("initSync", initSync). + Msg(WrapStagedSyncMsg("sync loop failed")) + go func() { + time.Sleep(5 * time.Second) + trigger() + }() + time.Sleep(1 * time.Second) + continue + } + d.logger.Info().Int("block added", addedBN). + Uint64("current height", d.bc.CurrentBlock().NumberU64()). + Bool("initSync", initSync). + Uint32("shard", d.bc.ShardID()). + Msg(WrapStagedSyncMsg("sync finished")) + + if addedBN != 0 { + // If block number has been changed, trigger another sync + // and try to add last mile from pub-sub (blocking) + go trigger() + if d.bh != nil { + d.bh.insertSync() + } + } + d.stagedSyncInstance.initSync = false + initSync = false + + case <-d.closeC: + return + } + } +} + +var emptySigVerifyErr *sigVerifyErr + +type sigVerifyErr struct { + err error +} + +func (e *sigVerifyErr) Error() string { + return fmt.Sprintf("[VerifyHeaderSignature] %v", e.err.Error()) +} + +func verifyAndInsertBlocks(bc blockChain, blocks types.Blocks) (int, error) { + for i, block := range blocks { + if err := verifyAndInsertBlock(bc, block, blocks[i+1:]...); err != nil { + return i, err + } + } + return len(blocks), nil +} + +func verifyAndInsertBlock(bc blockChain, block *types.Block, nextBlocks ...*types.Block) error { + var ( + sigBytes bls.SerializedSignature + bitmap []byte + err error + ) + if len(nextBlocks) > 0 { + // get commit sig from the next block + next := nextBlocks[0] + sigBytes = next.Header().LastCommitSignature() + bitmap = next.Header().LastCommitBitmap() + } else { + // get commit sig from current block + sigBytes, bitmap, err = chain.ParseCommitSigAndBitmap(block.GetCurrentCommitSig()) + if err != nil { + return errors.Wrap(err, "parse commitSigAndBitmap") + } + } + + if err := bc.Engine().VerifyHeaderSignature(bc, block.Header(), sigBytes, bitmap); err != nil { + return &sigVerifyErr{err} + } + if err := bc.Engine().VerifyHeader(bc, block.Header(), true); err != nil { + return errors.Wrap(err, "[VerifyHeader]") + } + if _, err := bc.InsertChain(types.Blocks{block}, false); err != nil { + return errors.Wrap(err, "[InsertChain]") + } + return nil +} diff --git a/api/service/stagedstreamsync/downloaders.go b/api/service/stagedstreamsync/downloaders.go new file mode 100644 index 0000000000..7b5881f100 --- /dev/null +++ b/api/service/stagedstreamsync/downloaders.go @@ -0,0 +1,96 @@ +package stagedstreamsync + +import ( + "github.com/harmony-one/abool" + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/p2p" +) + +// Downloaders is the set of downloaders +type Downloaders struct { + ds map[uint32]*Downloader + active *abool.AtomicBool + + config Config +} + +// NewDownloaders creates Downloaders for sync of multiple blockchains +func NewDownloaders(host p2p.Host, bcs []core.BlockChain, config Config) *Downloaders { + ds := make(map[uint32]*Downloader) + + for _, bc := range bcs { + if bc == nil { + continue + } + if _, ok := ds[bc.ShardID()]; ok { + continue + } + ds[bc.ShardID()] = NewDownloader(host, bc, config) + } + return &Downloaders{ + ds: ds, + active: abool.New(), + config: config, + } +} + +// Start starts the downloaders +func (ds *Downloaders) Start() { + if ds.config.ServerOnly { + // Run in server only mode. Do not start downloaders. + return + } + ds.active.Set() + for _, d := range ds.ds { + d.Start() + } +} + +// Close closes the downloaders +func (ds *Downloaders) Close() { + if ds.config.ServerOnly { + // Run in server only mode. Downloaders not started. + return + } + ds.active.UnSet() + for _, d := range ds.ds { + d.Close() + } +} + +// DownloadAsync triggers a download +func (ds *Downloaders) DownloadAsync(shardID uint32) { + d, ok := ds.ds[shardID] + if !ok && d != nil { + d.DownloadAsync() + } +} + +// GetShardDownloader returns the downloader with the given shard ID +func (ds *Downloaders) GetShardDownloader(shardID uint32) *Downloader { + return ds.ds[shardID] +} + +// NumPeers returns the connected peers for each shard +func (ds *Downloaders) NumPeers() map[uint32]int { + res := make(map[uint32]int) + + for sid, d := range ds.ds { + res[sid] = d.NumPeers() + } + return res +} + +// SyncStatus returns whether the given shard is doing syncing task and the target block number +func (ds *Downloaders) SyncStatus(shardID uint32) (bool, uint64, uint64) { + d, ok := ds.ds[shardID] + if !ok { + return false, 0, 0 + } + return d.SyncStatus() +} + +// IsActive returns whether the downloader is active +func (ds *Downloaders) IsActive() bool { + return ds.active.IsSet() +} diff --git a/api/service/stagedstreamsync/errors.go b/api/service/stagedstreamsync/errors.go new file mode 100644 index 0000000000..e4828003d2 --- /dev/null +++ b/api/service/stagedstreamsync/errors.go @@ -0,0 +1,56 @@ +package stagedstreamsync + +import ( + "fmt" +) + +// Errors ... +var ( + ErrRegistrationFail = WrapStagedSyncError("registration failed") + ErrGetBlock = WrapStagedSyncError("get block failed") + ErrGetBlockHash = WrapStagedSyncError("get block hash failed") + ErrGetConsensusHashes = WrapStagedSyncError("get consensus hashes failed") + ErrGenStateSyncTaskQueue = WrapStagedSyncError("generate state sync task queue failed") + ErrDownloadBlocks = WrapStagedSyncError("get download blocks failed") + ErrUpdateBlockAndStatus = WrapStagedSyncError("update block and status failed") + ErrGenerateNewState = WrapStagedSyncError("get generate new state failed") + ErrFetchBlockHashProgressFail = WrapStagedSyncError("fetch cache progress for block hashes stage failed") + ErrFetchCachedBlockHashFail = WrapStagedSyncError("fetch cached block hashes failed") + ErrNotEnoughBlockHashes = WrapStagedSyncError("peers haven't sent all requested block hashes") + ErrRetrieveCachedProgressFail = WrapStagedSyncError("retrieving cache progress for block hashes stage failed") + ErrRetrieveCachedHashProgressFail = WrapStagedSyncError("retrieving cache progress for block hashes stage failed") + ErrSaveBlockHashesProgressFail = WrapStagedSyncError("saving progress for block hashes stage failed") + ErrSaveCachedBlockHashesProgressFail = WrapStagedSyncError("saving cache progress for block hashes stage failed") + ErrSavingCacheLastBlockHashFail = WrapStagedSyncError("saving cache last block hash for block hashes stage failed") + ErrCachingBlockHashFail = WrapStagedSyncError("caching downloaded block hashes failed") + ErrCommitTransactionFail = WrapStagedSyncError("failed to write db commit") + ErrUnexpectedNumberOfBlocks = WrapStagedSyncError("unexpected number of block delivered") + ErrSavingBodiesProgressFail = WrapStagedSyncError("saving progress for block bodies stage failed") + ErrAddTasksToQueueFail = WrapStagedSyncError("cannot add task to queue") + ErrSavingCachedBodiesProgressFail = WrapStagedSyncError("saving cache progress for blocks stage failed") + ErrRetrievingCachedBodiesProgressFail = WrapStagedSyncError("retrieving cache progress for blocks stage failed") + ErrNoConnectedPeers = WrapStagedSyncError("haven't connected to any peer yet!") + ErrNotEnoughConnectedPeers = WrapStagedSyncError("not enough connected peers") + ErrSaveStateProgressFail = WrapStagedSyncError("saving progress for block States stage failed") + ErrPruningCursorCreationFail = WrapStagedSyncError("failed to create cursor for pruning") + ErrInvalidBlockNumber = WrapStagedSyncError("invalid block number") + ErrInvalidBlockBytes = WrapStagedSyncError("invalid block bytes to insert into chain") + ErrAddTaskFailed = WrapStagedSyncError("cannot add task to queue") + ErrNodeNotEnoughBlockHashes = WrapStagedSyncError("some of the nodes didn't provide all block hashes") + ErrCachingBlocksFail = WrapStagedSyncError("caching downloaded block bodies failed") + ErrSaveBlocksFail = WrapStagedSyncError("save downloaded block bodies failed") + ErrStageNotFound = WrapStagedSyncError("stage not found") + ErrSomeNodesNotReady = WrapStagedSyncError("some nodes are not ready") + ErrSomeNodesBlockHashFail = WrapStagedSyncError("some nodes failed to download block hashes") + ErrMaxPeerHeightFail = WrapStagedSyncError("get max peer height failed") +) + +// WrapStagedSyncError wraps errors for staged sync and returns error object +func WrapStagedSyncError(context string) error { + return fmt.Errorf("[STAGED_STREAM_SYNC]: %s", context) +} + +// WrapStagedSyncMsg wraps message for staged sync and returns string +func WrapStagedSyncMsg(context string) string { + return fmt.Sprintf("[STAGED_STREAM_SYNC]: %s", context) +} diff --git a/api/service/stagedstreamsync/helpers.go b/api/service/stagedstreamsync/helpers.go new file mode 100644 index 0000000000..cd6fd8f6f0 --- /dev/null +++ b/api/service/stagedstreamsync/helpers.go @@ -0,0 +1,114 @@ +package stagedstreamsync + +import ( + "encoding/binary" + "fmt" + "math" + + "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/harmony/core/types" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/pkg/errors" +) + +func marshalData(blockNumber uint64) []byte { + return encodeBigEndian(blockNumber) +} + +func unmarshalData(data []byte) (uint64, error) { + if len(data) == 0 { + return 0, nil + } + if len(data) < 8 { + return 0, fmt.Errorf("value must be at least 8 bytes, got %d", len(data)) + } + return binary.BigEndian.Uint64(data[:8]), nil +} + +func encodeBigEndian(n uint64) []byte { + var v [8]byte + binary.BigEndian.PutUint64(v[:], n) + return v[:] +} + +func divideCeil(x, y int) int { + fVal := float64(x) / float64(y) + return int(math.Ceil(fVal)) +} + +// computeBlockNumberByMaxVote computes the target block number by max vote. +func computeBlockNumberByMaxVote(votes map[sttypes.StreamID]uint64) uint64 { + var ( + nm = make(map[uint64]int) + res uint64 + maxCnt int + ) + for _, bn := range votes { + _, ok := nm[bn] + if !ok { + nm[bn] = 0 + } + nm[bn]++ + cnt := nm[bn] + + if cnt > maxCnt || (cnt == maxCnt && bn > res) { + res = bn + maxCnt = cnt + } + } + return res +} + +func checkGetBlockByHashesResult(blocks []*types.Block, hashes []common.Hash) error { + if len(blocks) != len(hashes) { + return errors.New("unexpected number of getBlocksByHashes result") + } + for i, block := range blocks { + if block == nil { + return errors.New("nil block found") + } + if block.Hash() != hashes[i] { + return fmt.Errorf("unexpected block hash: %x / %x", block.Hash(), hashes[i]) + } + } + return nil +} + +func countHashMaxVote(m map[sttypes.StreamID]common.Hash, whitelist map[sttypes.StreamID]struct{}) (common.Hash, map[sttypes.StreamID]struct{}) { + var ( + voteM = make(map[common.Hash]int) + res common.Hash + maxCnt = 0 + ) + + for st, h := range m { + if len(whitelist) != 0 { + if _, ok := whitelist[st]; !ok { + continue + } + } + if _, ok := voteM[h]; !ok { + voteM[h] = 0 + } + voteM[h]++ + if voteM[h] > maxCnt { + maxCnt = voteM[h] + res = h + } + } + + nextWl := make(map[sttypes.StreamID]struct{}) + for st, h := range m { + if h != res { + continue + } + if len(whitelist) != 0 { + if _, ok := whitelist[st]; ok { + nextWl[st] = struct{}{} + } + } else { + nextWl[st] = struct{}{} + } + } + return res, nextWl +} diff --git a/api/service/stagedstreamsync/metric.go b/api/service/stagedstreamsync/metric.go new file mode 100644 index 0000000000..1bc11a227f --- /dev/null +++ b/api/service/stagedstreamsync/metric.go @@ -0,0 +1,98 @@ +package stagedstreamsync + +import ( + "fmt" + + prom "github.com/harmony-one/harmony/api/service/prometheus" + "github.com/prometheus/client_golang/prometheus" +) + +func init() { + prom.PromRegistry().MustRegister( + consensusTriggeredDownloadCounterVec, + longRangeSyncedBlockCounterVec, + longRangeFailInsertedBlockCounterVec, + numShortRangeCounterVec, + numFailedDownloadCounterVec, + numBlocksInsertedShortRangeHistogramVec, + numBlocksInsertedBeaconHelperCounter, + ) +} + +var ( + consensusTriggeredDownloadCounterVec = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "hmy", + Subsystem: "downloader", + Name: "consensus_trigger", + Help: "number of times consensus triggered download task", + }, + []string{"ShardID"}, + ) + + longRangeSyncedBlockCounterVec = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "hmy", + Subsystem: "downloader", + Name: "num_blocks_synced_long_range", + Help: "number of blocks synced in long range sync", + }, + []string{"ShardID"}, + ) + + longRangeFailInsertedBlockCounterVec = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "hmy", + Subsystem: "downloader", + Name: "num_blocks_failed_long_range", + Help: "number of blocks failed to insert into change in long range sync", + }, + []string{"ShardID", "error"}, + ) + + numShortRangeCounterVec = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "hmy", + Subsystem: "downloader", + Name: "num_short_range", + Help: "number of short range sync is triggered", + }, + []string{"ShardID"}, + ) + + numFailedDownloadCounterVec = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "hmy", + Subsystem: "downloader", + Name: "failed_download", + Help: "number of downloading is failed", + }, + []string{"ShardID", "error"}, + ) + + numBlocksInsertedShortRangeHistogramVec = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "hmy", + Subsystem: "downloader", + Name: "num_blocks_inserted_short_range", + Help: "number of blocks inserted for each short range sync", + // Buckets: 0, 1, 2, 4, +INF (capped at 10) + Buckets: prometheus.ExponentialBuckets(0.5, 2, 5), + }, + []string{"ShardID"}, + ) + + numBlocksInsertedBeaconHelperCounter = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "hmy", + Subsystem: "downloader", + Name: "num_blocks_inserted_beacon_helper", + Help: "number of blocks inserted from beacon helper", + }, + ) +) + +func (d *Downloader) promLabels() prometheus.Labels { + sid := d.bc.ShardID() + return prometheus.Labels{"ShardID": fmt.Sprintf("%d", sid)} +} diff --git a/api/service/stagedstreamsync/service.go b/api/service/stagedstreamsync/service.go new file mode 100644 index 0000000000..46b182fb53 --- /dev/null +++ b/api/service/stagedstreamsync/service.go @@ -0,0 +1,30 @@ +package stagedstreamsync + +import ( + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/p2p" +) + +// StagedStreamSyncService is simply a adapter of downloaders, which support block synchronization +type StagedStreamSyncService struct { + Downloaders *Downloaders +} + +// NewService creates a new downloader service +func NewService(host p2p.Host, bcs []core.BlockChain, config Config) *StagedStreamSyncService { + return &StagedStreamSyncService{ + Downloaders: NewDownloaders(host, bcs, config), + } +} + +// Start starts the service +func (s *StagedStreamSyncService) Start() error { + s.Downloaders.Start() + return nil +} + +// Stop stops the service +func (s *StagedStreamSyncService) Stop() error { + s.Downloaders.Close() + return nil +} diff --git a/api/service/stagedstreamsync/short_range_helper.go b/api/service/stagedstreamsync/short_range_helper.go new file mode 100644 index 0000000000..f8dd2d28ea --- /dev/null +++ b/api/service/stagedstreamsync/short_range_helper.go @@ -0,0 +1,221 @@ +package stagedstreamsync + +import ( + "context" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/harmony/core/types" + syncProto "github.com/harmony-one/harmony/p2p/stream/protocols/sync" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/pkg/errors" + "github.com/rs/zerolog" +) + +type srHelper struct { + syncProtocol syncProtocol + + ctx context.Context + config Config + logger zerolog.Logger +} + +func (sh *srHelper) getHashChain(bns []uint64) ([]common.Hash, []sttypes.StreamID, error) { + results := newBlockHashResults(bns) + + var wg sync.WaitGroup + wg.Add(sh.config.Concurrency) + + for i := 0; i != sh.config.Concurrency; i++ { + go func(index int) { + defer wg.Done() + + hashes, stid, err := sh.doGetBlockHashesRequest(bns) + if err != nil { + sh.logger.Warn().Err(err).Str("StreamID", string(stid)). + Msg(WrapStagedSyncMsg("doGetBlockHashes return error")) + return + } + results.addResult(hashes, stid) + }(i) + } + wg.Wait() + + select { + case <-sh.ctx.Done(): + sh.logger.Info().Err(sh.ctx.Err()).Int("num blocks", results.numBlocksWithResults()). + Msg(WrapStagedSyncMsg("short range sync get hashes timed out")) + return nil, nil, sh.ctx.Err() + default: + } + + hashChain, wl := results.computeLongestHashChain() + sh.logger.Info().Int("hashChain size", len(hashChain)).Int("whitelist", len(wl)). + Msg(WrapStagedSyncMsg("computeLongestHashChain result")) + return hashChain, wl, nil +} + +func (sh *srHelper) getBlocksChain(bns []uint64) ([]*types.Block, sttypes.StreamID, error) { + return sh.doGetBlocksByNumbersRequest(bns) +} + +func (sh *srHelper) getBlocksByHashes(hashes []common.Hash, whitelist []sttypes.StreamID) ([]*types.Block, []sttypes.StreamID, error) { + ctx, cancel := context.WithCancel(sh.ctx) + defer cancel() + m := newGetBlocksByHashManager(hashes, whitelist) + + var ( + wg sync.WaitGroup + gErr error + errLock sync.Mutex + ) + + concurrency := sh.config.Concurrency + if concurrency > m.numRequests() { + concurrency = m.numRequests() + } + + wg.Add(concurrency) + for i := 0; i != concurrency; i++ { + go func(index int) { + defer wg.Done() + defer cancel() // it's ok to cancel context more than once + + for { + if m.isDone() { + return + } + hashes, wl, err := m.getNextHashes() + if err != nil { + errLock.Lock() + gErr = err + errLock.Unlock() + return + } + if len(hashes) == 0 { + select { + case <-time.After(200 * time.Millisecond): + continue + case <-ctx.Done(): + return + } + } + blocks, stid, err := sh.doGetBlocksByHashesRequest(ctx, hashes, wl) + if err != nil { + sh.logger.Warn().Err(err). + Str("StreamID", string(stid)). + Int("hashes", len(hashes)). + Int("index", index). + Msg(WrapStagedSyncMsg("getBlocksByHashes worker failed")) + m.handleResultError(hashes, stid) + } else { + m.addResult(hashes, blocks, stid) + } + } + }(i) + } + wg.Wait() + + if gErr != nil { + return nil, nil, gErr + } + select { + case <-sh.ctx.Done(): + res, _, _ := m.getResults() + sh.logger.Info().Err(sh.ctx.Err()).Int("num blocks", len(res)). + Msg(WrapStagedSyncMsg("short range sync get blocks timed out")) + return nil, nil, sh.ctx.Err() + default: + } + + return m.getResults() +} + +func (sh *srHelper) checkPrerequisites() error { + if sh.syncProtocol.NumStreams() < sh.config.Concurrency { + return errors.New("not enough streams") + } + return nil +} + +func (sh *srHelper) prepareBlockHashNumbers(curNumber uint64, count int) []uint64 { + + n := count + if count > BlockHashesPerRequest { + n = BlockHashesPerRequest + } + res := make([]uint64, 0, n) + + for bn := curNumber + 1; bn <= curNumber+uint64(n); bn++ { + res = append(res, bn) + } + return res +} + +func (sh *srHelper) doGetBlockHashesRequest(bns []uint64) ([]common.Hash, sttypes.StreamID, error) { + ctx, cancel := context.WithTimeout(sh.ctx, 1*time.Second) + defer cancel() + + hashes, stid, err := sh.syncProtocol.GetBlockHashes(ctx, bns) + if err != nil { + sh.logger.Warn().Err(err). + Interface("block numbers", bns). + Str("stream", string(stid)). + Msg(WrapStagedSyncMsg("failed to doGetBlockHashesRequest")) + return nil, stid, err + } + if len(hashes) != len(bns) { + err := errors.New("unexpected get block hashes result delivered") + sh.logger.Warn().Err(err). + Str("stream", string(stid)). + Msg(WrapStagedSyncMsg("failed to doGetBlockHashesRequest")) + sh.syncProtocol.StreamFailed(stid, "unexpected get block hashes result delivered") + return nil, stid, err + } + return hashes, stid, nil +} + +func (sh *srHelper) doGetBlocksByNumbersRequest(bns []uint64) ([]*types.Block, sttypes.StreamID, error) { + ctx, cancel := context.WithTimeout(sh.ctx, 10*time.Second) + defer cancel() + + blocks, stid, err := sh.syncProtocol.GetBlocksByNumber(ctx, bns) + if err != nil { + sh.logger.Warn().Err(err).Str("stream", string(stid)).Msg(WrapStagedSyncMsg("failed to doGetBlockHashesRequest")) + return nil, stid, err + } + return blocks, stid, nil +} + +func (sh *srHelper) doGetBlocksByHashesRequest(ctx context.Context, hashes []common.Hash, wl []sttypes.StreamID) ([]*types.Block, sttypes.StreamID, error) { + ctx, cancel := context.WithTimeout(sh.ctx, 10*time.Second) + defer cancel() + + blocks, stid, err := sh.syncProtocol.GetBlocksByHashes(ctx, hashes, + syncProto.WithWhitelist(wl)) + if err != nil { + sh.logger.Warn().Err(err).Str("stream", string(stid)).Msg("failed to getBlockByHashes") + return nil, stid, err + } + if err := checkGetBlockByHashesResult(blocks, hashes); err != nil { + sh.logger.Warn().Err(err).Str("stream", string(stid)).Msg(WrapStagedSyncMsg("failed to getBlockByHashes")) + sh.syncProtocol.StreamFailed(stid, "failed to getBlockByHashes") + return nil, stid, err + } + return blocks, stid, nil +} + +func (sh *srHelper) removeStreams(sts []sttypes.StreamID) { + for _, st := range sts { + sh.syncProtocol.RemoveStream(st) + } +} + +// blameAllStreams only not to blame all whitelisted streams when the it's not the last block signature verification failed. +func (sh *srHelper) blameAllStreams(blocks types.Blocks, errIndex int, err error) bool { + if errors.As(err, &emptySigVerifyErr) && errIndex == len(blocks)-1 { + return false + } + return true +} diff --git a/api/service/stagedstreamsync/stage.go b/api/service/stagedstreamsync/stage.go new file mode 100644 index 0000000000..8f2040a789 --- /dev/null +++ b/api/service/stagedstreamsync/stage.go @@ -0,0 +1,112 @@ +package stagedstreamsync + +import ( + "context" + "errors" + + "github.com/ethereum/go-ethereum/common" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/ledgerwatch/erigon-lib/kv" +) + +type ExecFunc func(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error + +type StageHandler interface { + // Exec is the execution function for the stage to move forward. + // * firstCycle - is it the first cycle of syncing. + // * invalidBlockRevert - whether the execution is to solve the invalid block + // * s - is the current state of the stage and contains stage data. + // * reverter - if the stage needs to cause reverting, `reverter` methods can be used. + Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error + + // Revert is the reverting logic of the stage. + // * firstCycle - is it the first cycle of syncing. + // * u - contains information about the revert itself. + // * s - represents the state of this stage at the beginning of revert. + Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) error + + // CleanUp is the execution function for the stage to prune old data. + // * firstCycle - is it the first cycle of syncing. + // * p - is the current state of the stage and contains stage data. + CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) error + + // SetStageContext updates the context for stage + SetStageContext(ctx context.Context) +} + +// Stage is a single sync stage in staged sync. +type Stage struct { + // ID of the sync stage. Should not be empty and should be unique. It is recommended to prefix it with reverse domain to avoid clashes (`com.example.my-stage`). + ID SyncStageID + // Handler handles the logic for the stage + Handler StageHandler + // Description is a string that is shown in the logs. + Description string + // DisabledDescription shows in the log with a message if the stage is disabled. Here, you can show which command line flags should be provided to enable the page. + DisabledDescription string + // Disabled defines if the stage is disabled. It sets up when the stage is build by its `StageBuilder`. + Disabled bool +} + +var ErrStopped = errors.New("stopped") +var ErrRevert = errors.New("unwound") + +// StageState is the state of the stage. +type StageState struct { + state *StagedStreamSync + ID SyncStageID + BlockNumber uint64 // BlockNumber is the current block number of the stage at the beginning of the state execution. +} + +func (s *StageState) LogPrefix() string { return s.state.LogPrefix() } + +func (s *StageState) CurrentStageProgress(db kv.Getter) (uint64, error) { + return GetStageProgress(db, s.ID, s.state.isBeacon) +} + +func (s *StageState) StageProgress(db kv.Getter, id SyncStageID) (uint64, error) { + return GetStageProgress(db, id, s.state.isBeacon) +} + +// Update updates the stage state (current block number) in the database. Can be called multiple times during stage execution. +func (s *StageState) Update(db kv.Putter, newBlockNum uint64) error { + return SaveStageProgress(db, s.ID, s.state.isBeacon, newBlockNum) +} +func (s *StageState) UpdateCleanUp(db kv.Putter, blockNum uint64) error { + return SaveStageCleanUpProgress(db, s.ID, s.state.isBeacon, blockNum) +} + +// Reverter allows the stage to cause an revert. +type Reverter interface { + // RevertTo begins staged sync revert to the specified block. + RevertTo(revertPoint uint64, invalidBlockNumber uint64, invalidBlockHash common.Hash, invalidBlockStreamID sttypes.StreamID) +} + +// RevertState contains the information about revert. +type RevertState struct { + ID SyncStageID + RevertPoint uint64 // RevertPoint is the block to revert to. + state *StagedStreamSync +} + +func (u *RevertState) LogPrefix() string { return u.state.LogPrefix() } + +// Done updates the DB state of the stage. +func (u *RevertState) Done(db kv.Putter) error { + return SaveStageProgress(db, u.ID, u.state.isBeacon, u.RevertPoint) +} + +type CleanUpState struct { + ID SyncStageID + ForwardProgress uint64 // progress of stage forward move + CleanUpProgress uint64 // progress of stage prune move. after sync cycle it become equal to ForwardProgress by Done() method + state *StagedStreamSync +} + +func (s *CleanUpState) LogPrefix() string { return s.state.LogPrefix() + " CleanUp" } +func (s *CleanUpState) Done(db kv.Putter) error { + return SaveStageCleanUpProgress(db, s.ID, s.state.isBeacon, s.ForwardProgress) +} +func (s *CleanUpState) DoneAt(db kv.Putter, blockNum uint64) error { + return SaveStageCleanUpProgress(db, s.ID, s.state.isBeacon, blockNum) +} diff --git a/api/service/stagedstreamsync/stage_bodies.go b/api/service/stagedstreamsync/stage_bodies.go new file mode 100644 index 0000000000..1f122b0d9c --- /dev/null +++ b/api/service/stagedstreamsync/stage_bodies.go @@ -0,0 +1,420 @@ +package stagedstreamsync + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/internal/utils" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/pkg/errors" +) + +type StageBodies struct { + configs StageBodiesCfg +} +type StageBodiesCfg struct { + ctx context.Context + bc core.BlockChain + db kv.RwDB + blockDBs []kv.RwDB + concurrency int + protocol syncProtocol + isBeacon bool + logProgress bool +} + +func NewStageBodies(cfg StageBodiesCfg) *StageBodies { + return &StageBodies{ + configs: cfg, + } +} + +func NewStageBodiesCfg(ctx context.Context, bc core.BlockChain, db kv.RwDB, blockDBs []kv.RwDB, concurrency int, protocol syncProtocol, isBeacon bool, logProgress bool) StageBodiesCfg { + return StageBodiesCfg{ + ctx: ctx, + bc: bc, + db: db, + blockDBs: blockDBs, + concurrency: concurrency, + protocol: protocol, + isBeacon: isBeacon, + logProgress: logProgress, + } +} + +func (b *StageBodies) SetStageContext(ctx context.Context) { + b.configs.ctx = ctx +} + +// Exec progresses Bodies stage in the forward direction +func (b *StageBodies) Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) (err error) { + + useInternalTx := tx == nil + + if invalidBlockRevert { + return b.redownloadBadBlock(s) + } + + // for short range sync, skip this stage + if !s.state.initSync { + return nil + } + + maxHeight := s.state.status.targetBN + currentHead := b.configs.bc.CurrentBlock().NumberU64() + if currentHead >= maxHeight { + return nil + } + currProgress := uint64(0) + targetHeight := s.state.currentCycle.TargetHeight + // isBeacon := s.state.isBeacon + // isLastCycle := targetHeight >= maxHeight + + if errV := CreateView(b.configs.ctx, b.configs.db, tx, func(etx kv.Tx) error { + if currProgress, err = s.CurrentStageProgress(etx); err != nil { + return err + } + return nil + }); errV != nil { + return errV + } + + if currProgress == 0 { + if err := b.cleanAllBlockDBs(); err != nil { + return err + } + currProgress = currentHead + } + + if currProgress >= targetHeight { + return nil + } + + // size := uint64(0) + startTime := time.Now() + // startBlock := currProgress + if b.configs.logProgress { + fmt.Print("\033[s") // save the cursor position + } + + if useInternalTx { + var err error + tx, err = b.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + // Fetch blocks from neighbors + s.state.gbm = newBlockDownloadManager(tx, b.configs.bc, targetHeight, s.state.logger) + + // Setup workers to fetch blocks from remote node + var wg sync.WaitGroup + + for i := 0; i != s.state.config.Concurrency; i++ { + wg.Add(1) + go b.runBlockWorkerLoop(s.state.gbm, &wg, i, startTime) + } + + wg.Wait() + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + + return nil +} + +// runBlockWorkerLoop creates a work loop for download blocks +func (b *StageBodies) runBlockWorkerLoop(gbm *blockDownloadManager, wg *sync.WaitGroup, loopID int, startTime time.Time) { + + currentBlock := int(b.configs.bc.CurrentBlock().NumberU64()) + + defer wg.Done() + + for { + select { + case <-b.configs.ctx.Done(): + return + default: + } + batch := gbm.GetNextBatch() + if len(batch) == 0 { + select { + case <-b.configs.ctx.Done(): + return + case <-time.After(100 * time.Millisecond): + return + } + } + + blockBytes, sigBytes, stid, err := b.downloadRawBlocks(batch) + if err != nil { + if !errors.Is(err, context.Canceled) { + b.configs.protocol.StreamFailed(stid, "downloadRawBlocks failed") + } + utils.Logger().Error(). + Err(err). + Str("stream", string(stid)). + Interface("block numbers", batch). + Msg(WrapStagedSyncMsg("downloadRawBlocks failed")) + err = errors.Wrap(err, "request error") + gbm.HandleRequestError(batch, err, stid) + } else { + if err = b.saveBlocks(gbm.tx, batch, blockBytes, sigBytes, loopID, stid); err != nil { + panic("[STAGED_STREAM_SYNC] saving downloaded blocks to db failed.") + } + gbm.HandleRequestResult(batch, blockBytes, sigBytes, loopID, stid) + if b.configs.logProgress { + //calculating block download speed + dt := time.Now().Sub(startTime).Seconds() + speed := float64(0) + if dt > 0 { + speed = float64(len(gbm.bdd)) / dt + } + blockSpeed := fmt.Sprintf("%.2f", speed) + + fmt.Print("\033[u\033[K") // restore the cursor position and clear the line + fmt.Println("downloaded blocks:", currentBlock+len(gbm.bdd), "/", int(gbm.targetBN), "(", blockSpeed, "blocks/s", ")") + } + } + } +} + +// redownloadBadBlock tries to redownload the bad block from other streams +func (b *StageBodies) redownloadBadBlock(s *StageState) error { + + batch := make([]uint64, 1) + batch = append(batch, s.state.invalidBlock.Number) + + for { + if b.configs.protocol.NumStreams() == 0 { + return errors.Errorf("re-download bad block from all streams failed") + } + blockBytes, sigBytes, stid, err := b.downloadRawBlocks(batch) + if err != nil { + if !errors.Is(err, context.Canceled) { + b.configs.protocol.StreamFailed(stid, "tried to re-download bad block from this stream, but downloadRawBlocks failed") + } + continue + } + isOneOfTheBadStreams := false + for _, id := range s.state.invalidBlock.StreamID { + if id == stid { + b.configs.protocol.RemoveStream(stid) + isOneOfTheBadStreams = true + break + } + } + if isOneOfTheBadStreams { + continue + } + s.state.gbm.SetDownloadDetails(batch, 0, stid) + if errU := b.configs.blockDBs[0].Update(context.Background(), func(tx kv.RwTx) error { + if err = b.saveBlocks(tx, batch, blockBytes, sigBytes, 0, stid); err != nil { + return errors.Errorf("[STAGED_STREAM_SYNC] saving re-downloaded bad block to db failed.") + } + return nil + }); errU != nil { + continue + } + break + } + return nil +} + +func (b *StageBodies) downloadBlocks(bns []uint64) ([]*types.Block, sttypes.StreamID, error) { + ctx, cancel := context.WithTimeout(b.configs.ctx, 10*time.Second) + defer cancel() + + blocks, stid, err := b.configs.protocol.GetBlocksByNumber(ctx, bns) + if err != nil { + return nil, stid, err + } + if err := validateGetBlocksResult(bns, blocks); err != nil { + return nil, stid, err + } + return blocks, stid, nil +} + +func (b *StageBodies) downloadRawBlocks(bns []uint64) ([][]byte, [][]byte, sttypes.StreamID, error) { + ctx, cancel := context.WithTimeout(b.configs.ctx, 10*time.Second) + defer cancel() + + return b.configs.protocol.GetRawBlocksByNumber(ctx, bns) +} + +func validateGetBlocksResult(requested []uint64, result []*types.Block) error { + if len(result) != len(requested) { + return fmt.Errorf("unexpected number of blocks delivered: %v / %v", len(result), len(requested)) + } + for i, block := range result { + if block != nil && block.NumberU64() != requested[i] { + return fmt.Errorf("block with unexpected number delivered: %v / %v", block.NumberU64(), requested[i]) + } + } + return nil +} + +// saveBlocks saves the blocks into db +func (b *StageBodies) saveBlocks(tx kv.RwTx, bns []uint64, blockBytes [][]byte, sigBytes [][]byte, loopID int, stid sttypes.StreamID) error { + + tx, err := b.configs.blockDBs[loopID].BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + + for i := uint64(0); i < uint64(len(blockBytes)); i++ { + block := blockBytes[i] + sig := sigBytes[i] + if block == nil { + continue + } + + blkKey := marshalData(bns[i]) + + if err := tx.Put(BlocksBucket, blkKey, block); err != nil { + utils.Logger().Error(). + Err(err). + Uint64("block height", bns[i]). + Msg("[STAGED_STREAM_SYNC] adding block to db failed") + return err + } + // sigKey := []byte("s" + string(bns[i])) + if err := tx.Put(BlockSignaturesBucket, blkKey, sig); err != nil { + utils.Logger().Error(). + Err(err). + Uint64("block height", bns[i]). + Msg("[STAGED_STREAM_SYNC] adding block sig to db failed") + return err + } + } + + if err := tx.Commit(); err != nil { + return err + } + + return nil +} + +func (b *StageBodies) saveProgress(s *StageState, progress uint64, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + var err error + tx, err = b.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + // save progress + if err = s.Update(tx, progress); err != nil { + utils.Logger().Error(). + Err(err). + Msgf("[STAGED_SYNC] saving progress for block bodies stage failed") + return ErrSavingBodiesProgressFail + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + return nil +} + +func (b *StageBodies) cleanBlocksDB(loopID int) (err error) { + + tx, errb := b.configs.blockDBs[loopID].BeginRw(b.configs.ctx) + if errb != nil { + return errb + } + defer tx.Rollback() + + // clean block bodies db + if err = tx.ClearBucket(BlocksBucket); err != nil { + utils.Logger().Error(). + Err(err). + Msgf("[STAGED_STREAM_SYNC] clear blocks bucket after revert failed") + return err + } + // clean block signatures db + if err = tx.ClearBucket(BlockSignaturesBucket); err != nil { + utils.Logger().Error(). + Err(err). + Msgf("[STAGED_STREAM_SYNC] clear block signatures bucket after revert failed") + return err + } + + if err = tx.Commit(); err != nil { + return err + } + + return nil +} + +func (b *StageBodies) cleanAllBlockDBs() (err error) { + //clean all blocks DBs + for i := 0; i < b.configs.concurrency; i++ { + if err := b.cleanBlocksDB(i); err != nil { + return err + } + } + return nil +} + +func (b *StageBodies) Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { + + //clean all blocks DBs + if err := b.cleanAllBlockDBs(); err != nil { + return err + } + + useInternalTx := tx == nil + if useInternalTx { + tx, err = b.configs.db.BeginRw(b.configs.ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + // save progress + currentHead := b.configs.bc.CurrentBlock().NumberU64() + if err = s.Update(tx, currentHead); err != nil { + utils.Logger().Error(). + Err(err). + Msgf("[STAGED_SYNC] saving progress for block bodies stage after revert failed") + return err + } + + if err = u.Done(tx); err != nil { + return err + } + + if useInternalTx { + if err = tx.Commit(); err != nil { + return err + } + } + return nil +} + +func (b *StageBodies) CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { + + //clean all blocks DBs + if err := b.cleanAllBlockDBs(); err != nil { + return err + } + + return nil +} diff --git a/api/service/stagedstreamsync/stage_epoch.go b/api/service/stagedstreamsync/stage_epoch.go new file mode 100644 index 0000000000..6320d82175 --- /dev/null +++ b/api/service/stagedstreamsync/stage_epoch.go @@ -0,0 +1,198 @@ +package stagedstreamsync + +import ( + "context" + + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/shard" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/pkg/errors" +) + +type StageEpoch struct { + configs StageEpochCfg +} + +type StageEpochCfg struct { + ctx context.Context + bc core.BlockChain + db kv.RwDB +} + +func NewStageEpoch(cfg StageEpochCfg) *StageEpoch { + return &StageEpoch{ + configs: cfg, + } +} + +func NewStageEpochCfg(ctx context.Context, bc core.BlockChain, db kv.RwDB) StageEpochCfg { + return StageEpochCfg{ + ctx: ctx, + bc: bc, + db: db, + } +} + +func (sr *StageEpoch) SetStageContext(ctx context.Context) { + sr.configs.ctx = ctx +} + +func (sr *StageEpoch) Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error { + + // no need to update epoch chain if we are redoing the stages because of bad block + if invalidBlockRevert { + return nil + } + // for long range sync, skip this stage + if s.state.initSync { + return nil + } + + if _, ok := sr.configs.bc.(*core.EpochChain); !ok { + return nil + } + + // doShortRangeSyncForEpochSync + n, err := sr.doShortRangeSyncForEpochSync(s) + s.state.inserted = n + if err != nil { + return err + } + + useInternalTx := tx == nil + if useInternalTx { + var err error + tx, err = sr.configs.db.BeginRw(sr.configs.ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + + return nil +} + +func (sr *StageEpoch) doShortRangeSyncForEpochSync(s *StageState) (int, error) { + + numShortRangeCounterVec.With(s.state.promLabels()).Inc() + + srCtx, cancel := context.WithTimeout(s.state.ctx, ShortRangeTimeout) + defer cancel() + + //TODO: merge srHelper with StageEpochConfig + sh := &srHelper{ + syncProtocol: s.state.protocol, + ctx: srCtx, + config: s.state.config, + logger: utils.Logger().With().Str("mode", "epoch chain short range").Logger(), + } + + if err := sh.checkPrerequisites(); err != nil { + return 0, errors.Wrap(err, "prerequisite") + } + curBN := s.state.bc.CurrentBlock().NumberU64() + bns := make([]uint64, 0, BlocksPerRequest) + // in epoch chain, we have only the last block of each epoch, so, the current + // block's epoch number shows the last epoch we have. We should start + // from next epoch then + loopEpoch := s.state.bc.CurrentHeader().Epoch().Uint64() + 1 + for len(bns) < BlocksPerRequest { + blockNum := shard.Schedule.EpochLastBlock(loopEpoch) + if blockNum > curBN { + bns = append(bns, blockNum) + } + loopEpoch = loopEpoch + 1 + } + + if len(bns) == 0 { + return 0, nil + } + + //////////////////////////////////////////////////////// + hashChain, whitelist, err := sh.getHashChain(bns) + if err != nil { + return 0, errors.Wrap(err, "getHashChain") + } + if len(hashChain) == 0 { + // short circuit for no sync is needed + return 0, nil + } + blocks, streamID, err := sh.getBlocksByHashes(hashChain, whitelist) + if err != nil { + utils.Logger().Warn().Err(err).Msg("epoch sync getBlocksByHashes failed") + if !errors.Is(err, context.Canceled) { + sh.removeStreams(whitelist) // Remote nodes cannot provide blocks with target hashes + } + return 0, errors.Wrap(err, "epoch sync getBlocksByHashes") + } + /////////////////////////////////////////////////////// + // TODO: check this + // blocks, streamID, err := sh.getBlocksChain(bns) + // if err != nil { + // return 0, errors.Wrap(err, "getHashChain") + // } + /////////////////////////////////////////////////////// + if len(blocks) == 0 { + // short circuit for no sync is needed + return 0, nil + } + + n, err := s.state.bc.InsertChain(blocks, true) + numBlocksInsertedShortRangeHistogramVec.With(s.state.promLabels()).Observe(float64(n)) + if err != nil { + utils.Logger().Info().Err(err).Int("blocks inserted", n).Msg("Insert block failed") + sh.removeStreams(streamID) // Data provided by remote nodes is corrupted + return n, err + } + if n > 0 { + utils.Logger().Info().Int("blocks inserted", n).Msg("Insert block success") + } + return n, nil +} + +func (sr *StageEpoch) Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = sr.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + if err = u.Done(tx); err != nil { + return err + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + return nil +} + +func (sr *StageEpoch) CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = sr.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + if useInternalTx { + if err = tx.Commit(); err != nil { + return err + } + } + return nil +} diff --git a/api/service/stagedstreamsync/stage_finish.go b/api/service/stagedstreamsync/stage_finish.go new file mode 100644 index 0000000000..9039b5588d --- /dev/null +++ b/api/service/stagedstreamsync/stage_finish.go @@ -0,0 +1,114 @@ +package stagedstreamsync + +import ( + "context" + + "github.com/ledgerwatch/erigon-lib/kv" +) + +type StageFinish struct { + configs StageFinishCfg +} + +type StageFinishCfg struct { + ctx context.Context + db kv.RwDB +} + +func NewStageFinish(cfg StageFinishCfg) *StageFinish { + return &StageFinish{ + configs: cfg, + } +} + +func NewStageFinishCfg(ctx context.Context, db kv.RwDB) StageFinishCfg { + return StageFinishCfg{ + ctx: ctx, + db: db, + } +} + +func (finish *StageFinish) SetStageContext(ctx context.Context) { + finish.configs.ctx = ctx +} + +func (finish *StageFinish) Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error { + useInternalTx := tx == nil + if useInternalTx { + var err error + tx, err = finish.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + // TODO: prepare indices (useful for RPC) and finalize + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + + return nil +} + +func (bh *StageFinish) clearBucket(tx kv.RwTx, isBeacon bool) error { + useInternalTx := tx == nil + if useInternalTx { + var err error + tx, err = bh.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + return nil +} + +func (finish *StageFinish) Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = finish.configs.db.BeginRw(finish.configs.ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + if err = u.Done(tx); err != nil { + return err + } + + if useInternalTx { + if err = tx.Commit(); err != nil { + return err + } + } + return nil +} + +func (finish *StageFinish) CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = finish.configs.db.BeginRw(finish.configs.ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + if useInternalTx { + if err = tx.Commit(); err != nil { + return err + } + } + return nil +} diff --git a/api/service/stagedstreamsync/stage_heads.go b/api/service/stagedstreamsync/stage_heads.go new file mode 100644 index 0000000000..1ddd285aa0 --- /dev/null +++ b/api/service/stagedstreamsync/stage_heads.go @@ -0,0 +1,157 @@ +package stagedstreamsync + +import ( + "context" + + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/internal/utils" + "github.com/ledgerwatch/erigon-lib/kv" +) + +type StageHeads struct { + configs StageHeadsCfg +} + +type StageHeadsCfg struct { + ctx context.Context + bc core.BlockChain + db kv.RwDB +} + +func NewStageHeads(cfg StageHeadsCfg) *StageHeads { + return &StageHeads{ + configs: cfg, + } +} + +func NewStageHeadersCfg(ctx context.Context, bc core.BlockChain, db kv.RwDB) StageHeadsCfg { + return StageHeadsCfg{ + ctx: ctx, + bc: bc, + db: db, + } +} + +func (heads *StageHeads) SetStageContext(ctx context.Context) { + heads.configs.ctx = ctx +} + +func (heads *StageHeads) Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error { + + // no need to update target if we are redoing the stages because of bad block + if invalidBlockRevert { + return nil + } + + // no need for short range sync + if !s.state.initSync { + return nil + } + + useInternalTx := tx == nil + if useInternalTx { + var err error + tx, err = heads.configs.db.BeginRw(heads.configs.ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + maxHeight := s.state.status.targetBN + maxBlocksPerSyncCycle := uint64(1024) // TODO: should be in config -> s.state.MaxBlocksPerSyncCycle + currentHeight := heads.configs.bc.CurrentBlock().NumberU64() + s.state.currentCycle.TargetHeight = maxHeight + targetHeight := uint64(0) + if errV := CreateView(heads.configs.ctx, heads.configs.db, tx, func(etx kv.Tx) (err error) { + if targetHeight, err = s.CurrentStageProgress(etx); err != nil { + return err + } + return nil + }); errV != nil { + return errV + } + + if currentHeight >= maxHeight { + utils.Logger().Info().Uint64("current number", currentHeight).Uint64("target number", maxHeight). + Msg(WrapStagedSyncMsg("early return of long range sync")) + return nil + } + + // if current height is ahead of target height, we need recalculate target height + if currentHeight >= targetHeight { + if maxHeight <= currentHeight { + return nil + } + utils.Logger().Info(). + Uint64("max blocks per sync cycle", maxBlocksPerSyncCycle). + Uint64("maxPeersHeight", maxHeight). + Msgf(WrapStagedSyncMsg("current height is ahead of target height, target height is readjusted to max peers height")) + targetHeight = maxHeight + } + + if targetHeight > maxHeight { + targetHeight = maxHeight + } + + if maxBlocksPerSyncCycle > 0 && targetHeight-currentHeight > maxBlocksPerSyncCycle { + targetHeight = currentHeight + maxBlocksPerSyncCycle + } + + s.state.currentCycle.TargetHeight = targetHeight + + if err := s.Update(tx, targetHeight); err != nil { + utils.Logger().Error(). + Err(err). + Msgf(WrapStagedSyncMsg("saving progress for headers stage failed")) + return err + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + + return nil +} + +func (heads *StageHeads) Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = heads.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + if err = u.Done(tx); err != nil { + return err + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + return nil +} + +func (heads *StageHeads) CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = heads.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + if useInternalTx { + if err = tx.Commit(); err != nil { + return err + } + } + return nil +} diff --git a/api/service/stagedstreamsync/stage_short_range.go b/api/service/stagedstreamsync/stage_short_range.go new file mode 100644 index 0000000000..de83beafa3 --- /dev/null +++ b/api/service/stagedstreamsync/stage_short_range.go @@ -0,0 +1,205 @@ +package stagedstreamsync + +import ( + "context" + + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/internal/utils" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/pkg/errors" +) + +type StageShortRange struct { + configs StageShortRangeCfg +} + +type StageShortRangeCfg struct { + ctx context.Context + bc core.BlockChain + db kv.RwDB +} + +func NewStageShortRange(cfg StageShortRangeCfg) *StageShortRange { + return &StageShortRange{ + configs: cfg, + } +} + +func NewStageShortRangeCfg(ctx context.Context, bc core.BlockChain, db kv.RwDB) StageShortRangeCfg { + return StageShortRangeCfg{ + ctx: ctx, + bc: bc, + db: db, + } +} + +func (sr *StageShortRange) SetStageContext(ctx context.Context) { + sr.configs.ctx = ctx +} + +func (sr *StageShortRange) Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error { + + // no need to do short range if we are redoing the stages because of bad block + if invalidBlockRevert { + return nil + } + + // for long range sync, skip this stage + if s.state.initSync { + return nil + } + + if _, ok := sr.configs.bc.(*core.EpochChain); ok { + return nil + } + + curBN := sr.configs.bc.CurrentBlock().NumberU64() + if curBN >= s.state.status.targetBN { + return nil + } + + // do short range sync + n, err := sr.doShortRangeSync(s) + s.state.inserted = n + if err != nil { + return err + } + + useInternalTx := tx == nil + if useInternalTx { + var err error + tx, err = sr.configs.db.BeginRw(sr.configs.ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + + return nil +} + +// doShortRangeSync does the short range sync. +// Compared with long range sync, short range sync is more focused on syncing to the latest block. +// It consist of 3 steps: +// 1. Obtain the block hashes and compute the longest hash chain.. +// 2. Get blocks by hashes from computed hash chain. +// 3. Insert the blocks to blockchain. +func (sr *StageShortRange) doShortRangeSync(s *StageState) (int, error) { + + numShortRangeCounterVec.With(s.state.promLabels()).Inc() + + srCtx, cancel := context.WithTimeout(s.state.ctx, ShortRangeTimeout) + defer cancel() + + sh := &srHelper{ + syncProtocol: s.state.protocol, + ctx: srCtx, + config: s.state.config, + logger: utils.Logger().With().Str("mode", "short range").Logger(), + } + + if err := sh.checkPrerequisites(); err != nil { + return 0, errors.Wrap(err, "prerequisite") + } + curBN := sr.configs.bc.CurrentBlock().NumberU64() + blkCount := int(s.state.status.targetBN) - int(curBN) + blkNums := sh.prepareBlockHashNumbers(curBN, blkCount) + hashChain, whitelist, err := sh.getHashChain(blkNums) + if err != nil { + return 0, errors.Wrap(err, "getHashChain") + } + + if len(hashChain) == 0 { + // short circuit for no sync is needed + return 0, nil + } + + expEndBN := curBN + uint64(len(hashChain)) + utils.Logger().Info().Uint64("current number", curBN). + Uint64("target number", expEndBN). + Interface("hashChain", hashChain). + Msg("short range start syncing") + + s.state.status.setTargetBN(expEndBN) + + s.state.status.startSyncing() + defer func() { + utils.Logger().Info().Msg("short range finished syncing") + s.state.status.finishSyncing() + }() + + blocks, stids, err := sh.getBlocksByHashes(hashChain, whitelist) + if err != nil { + utils.Logger().Warn().Err(err).Msg("getBlocksByHashes failed") + if !errors.Is(err, context.Canceled) { + sh.removeStreams(whitelist) // Remote nodes cannot provide blocks with target hashes + } + return 0, errors.Wrap(err, "getBlocksByHashes") + } + + utils.Logger().Info().Int("num blocks", len(blocks)).Msg("getBlockByHashes result") + + n, err := verifyAndInsertBlocks(sr.configs.bc, blocks) + numBlocksInsertedShortRangeHistogramVec.With(s.state.promLabels()).Observe(float64(n)) + if err != nil { + utils.Logger().Warn().Err(err).Int("blocks inserted", n).Msg("Insert block failed") + if sh.blameAllStreams(blocks, n, err) { + sh.removeStreams(whitelist) // Data provided by remote nodes is corrupted + } else { + // It is the last block gives a wrong commit sig. Blame the provider of the last block. + st2Blame := stids[len(stids)-1] + sh.removeStreams([]sttypes.StreamID{st2Blame}) + } + return n, err + } + utils.Logger().Info().Err(err).Int("blocks inserted", n).Msg("Insert block success") + + return n, nil +} + +func (sr *StageShortRange) Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = sr.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + if err = u.Done(tx); err != nil { + return err + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + return nil +} + +func (sr *StageShortRange) CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = sr.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + if useInternalTx { + if err = tx.Commit(); err != nil { + return err + } + } + return nil +} diff --git a/api/service/stagedstreamsync/stage_state.go b/api/service/stagedstreamsync/stage_state.go new file mode 100644 index 0000000000..9eda042477 --- /dev/null +++ b/api/service/stagedstreamsync/stage_state.go @@ -0,0 +1,295 @@ +package stagedstreamsync + +import ( + "context" + "fmt" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/internal/utils" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/prometheus/client_golang/prometheus" + "github.com/rs/zerolog" +) + +type StageStates struct { + configs StageStatesCfg +} +type StageStatesCfg struct { + ctx context.Context + bc core.BlockChain + db kv.RwDB + blockDBs []kv.RwDB + concurrency int + logger zerolog.Logger + logProgress bool +} + +func NewStageStates(cfg StageStatesCfg) *StageStates { + return &StageStates{ + configs: cfg, + } +} + +func NewStageStatesCfg(ctx context.Context, + bc core.BlockChain, + db kv.RwDB, + blockDBs []kv.RwDB, + concurrency int, + logger zerolog.Logger, + logProgress bool) StageStatesCfg { + + return StageStatesCfg{ + ctx: ctx, + bc: bc, + db: db, + blockDBs: blockDBs, + concurrency: concurrency, + logger: logger, + logProgress: logProgress, + } +} + +func (stg *StageStates) SetStageContext(ctx context.Context) { + stg.configs.ctx = ctx +} + +// Exec progresses States stage in the forward direction +func (stg *StageStates) Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) (err error) { + + // for short range sync, skip this step + if !s.state.initSync { + return nil + } + + maxHeight := s.state.status.targetBN + currentHead := stg.configs.bc.CurrentBlock().NumberU64() + if currentHead >= maxHeight { + return nil + } + currProgress := stg.configs.bc.CurrentBlock().NumberU64() + targetHeight := s.state.currentCycle.TargetHeight + if currProgress >= targetHeight { + return nil + } + useInternalTx := tx == nil + if useInternalTx { + var err error + tx, err = stg.configs.db.BeginRw(stg.configs.ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + // isLastCycle := targetHeight >= maxHeight + startTime := time.Now() + startBlock := currProgress + pl := s.state.promLabels() + gbm := s.state.gbm + + // prepare db transactions + txs := make([]kv.RwTx, stg.configs.concurrency) + for i := 0; i < stg.configs.concurrency; i++ { + txs[i], err = stg.configs.blockDBs[i].BeginRw(context.Background()) + if err != nil { + return err + } + } + + defer func() { + for i := 0; i < stg.configs.concurrency; i++ { + txs[i].Rollback() + } + }() + + if stg.configs.logProgress { + fmt.Print("\033[s") // save the cursor position + } + + for i := currProgress + 1; i <= targetHeight; i++ { + blkKey := marshalData(i) + loopID, streamID := gbm.GetDownloadDetails(i) + + blockBytes, err := txs[loopID].GetOne(BlocksBucket, blkKey) + if err != nil { + return err + } + sigBytes, err := txs[loopID].GetOne(BlockSignaturesBucket, blkKey) + if err != nil { + return err + } + + // if block size is invalid, we have to break the updating state loop + // we don't need to do rollback, because the latest batch haven't added to chain yet + sz := len(blockBytes) + if sz <= 1 { + utils.Logger().Error(). + Uint64("block number", i). + Msg("block size invalid") + invalidBlockHash := common.Hash{} + s.state.protocol.StreamFailed(streamID, "zero bytes block is received from stream") + reverter.RevertTo(stg.configs.bc.CurrentBlock().NumberU64(), i, invalidBlockHash, streamID) + return ErrInvalidBlockBytes + } + + var block *types.Block + if err := rlp.DecodeBytes(blockBytes, &block); err != nil { + utils.Logger().Error(). + Uint64("block number", i). + Msg("block size invalid") + s.state.protocol.StreamFailed(streamID, "invalid block is received from stream") + invalidBlockHash := common.Hash{} + reverter.RevertTo(stg.configs.bc.CurrentBlock().NumberU64(), i, invalidBlockHash, streamID) + return ErrInvalidBlockBytes + } + if sigBytes != nil { + block.SetCurrentCommitSig(sigBytes) + } + + if block.NumberU64() != i { + s.state.protocol.StreamFailed(streamID, "invalid block with unmatched number is received from stream") + invalidBlockHash := block.Hash() + reverter.RevertTo(stg.configs.bc.CurrentBlock().NumberU64(), i, invalidBlockHash, streamID) + return ErrInvalidBlockNumber + } + + if err := verifyAndInsertBlock(stg.configs.bc, block); err != nil { + stg.configs.logger.Warn().Err(err).Uint64("cycle target block", targetHeight). + Uint64("block number", block.NumberU64()). + Msg(WrapStagedSyncMsg("insert blocks failed in long range")) + s.state.protocol.StreamFailed(streamID, "unverifiable invalid block is received from stream") + invalidBlockHash := block.Hash() + reverter.RevertTo(stg.configs.bc.CurrentBlock().NumberU64(), block.NumberU64(), invalidBlockHash, streamID) + pl["error"] = err.Error() + longRangeFailInsertedBlockCounterVec.With(pl).Inc() + return err + } + + if invalidBlockRevert { + if s.state.invalidBlock.Number == i { + s.state.invalidBlock.resolve() + } + } + + s.state.inserted++ + longRangeSyncedBlockCounterVec.With(pl).Inc() + + utils.Logger().Info(). + Uint64("blockHeight", block.NumberU64()). + Uint64("blockEpoch", block.Epoch().Uint64()). + Str("blockHex", block.Hash().Hex()). + Uint32("ShardID", block.ShardID()). + Msg("[STAGED_STREAM_SYNC] New Block Added to Blockchain") + + // update cur progress + currProgress = stg.configs.bc.CurrentBlock().NumberU64() + + for i, tx := range block.StakingTransactions() { + utils.Logger().Info(). + Msgf( + "StakingTxn %d: %s, %v", i, tx.StakingType().String(), tx.StakingMessage(), + ) + } + + // log the stage progress in console + if stg.configs.logProgress { + //calculating block speed + dt := time.Now().Sub(startTime).Seconds() + speed := float64(0) + if dt > 0 { + speed = float64(currProgress-startBlock) / dt + } + blockSpeed := fmt.Sprintf("%.2f", speed) + fmt.Print("\033[u\033[K") // restore the cursor position and clear the line + fmt.Println("insert blocks progress:", currProgress, "/", targetHeight, "(", blockSpeed, "blocks/s", ")") + } + + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + + return nil +} + +func (stg *StageStates) insertChain(gbm *blockDownloadManager, + protocol syncProtocol, + lbls prometheus.Labels, + targetBN uint64) { + +} + +func (stg *StageStates) saveProgress(s *StageState, tx kv.RwTx) (err error) { + + useInternalTx := tx == nil + if useInternalTx { + var err error + tx, err = stg.configs.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + // save progress + if err = s.Update(tx, stg.configs.bc.CurrentBlock().NumberU64()); err != nil { + utils.Logger().Error(). + Err(err). + Msgf("[STAGED_SYNC] saving progress for block States stage failed") + return ErrSaveStateProgressFail + } + + if useInternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + return nil +} + +func (stg *StageStates) Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = stg.configs.db.BeginRw(stg.configs.ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + if err = u.Done(tx); err != nil { + return err + } + + if useInternalTx { + if err = tx.Commit(); err != nil { + return err + } + } + return nil +} + +func (stg *StageStates) CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { + useInternalTx := tx == nil + if useInternalTx { + tx, err = stg.configs.db.BeginRw(stg.configs.ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + if useInternalTx { + if err = tx.Commit(); err != nil { + return err + } + } + return nil +} diff --git a/api/service/stagedstreamsync/staged_stream_sync.go b/api/service/stagedstreamsync/staged_stream_sync.go new file mode 100644 index 0000000000..1abd728498 --- /dev/null +++ b/api/service/stagedstreamsync/staged_stream_sync.go @@ -0,0 +1,597 @@ +package stagedstreamsync + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/event" + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/internal/utils" + syncproto "github.com/harmony-one/harmony/p2p/stream/protocols/sync" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/prometheus/client_golang/prometheus" + "github.com/rs/zerolog" +) + +type InvalidBlock struct { + Active bool + Number uint64 + Hash common.Hash + IsLogged bool + StreamID []sttypes.StreamID +} + +func (ib *InvalidBlock) set(num uint64, hash common.Hash, resetBadStreams bool) { + ib.Active = true + ib.IsLogged = false + ib.Number = num + ib.Hash = hash + if resetBadStreams { + ib.StreamID = make([]sttypes.StreamID, 0) + } +} + +func (ib *InvalidBlock) resolve() { + ib.Active = false + ib.IsLogged = false + ib.Number = 0 + ib.Hash = common.Hash{} + ib.StreamID = ib.StreamID[:0] +} + +func (ib *InvalidBlock) addBadStream(bsID sttypes.StreamID) { + // only add uniques IDs + for _, stID := range ib.StreamID { + if stID == bsID { + return + } + } + ib.StreamID = append(ib.StreamID, bsID) +} + +type StagedStreamSync struct { + ctx context.Context + bc core.BlockChain + isBeacon bool + isExplorer bool + db kv.RwDB + protocol syncProtocol + gbm *blockDownloadManager // initialized when finished get block number + inserted int + config Config + logger zerolog.Logger + status status //TODO: merge this with currentSyncCycle + initSync bool // if sets to true, node start long range syncing + UseMemDB bool + + revertPoint *uint64 // used to run stages + prevRevertPoint *uint64 // used to get value from outside of staged sync after cycle (for example to notify RPCDaemon) + invalidBlock InvalidBlock + currentStage uint + LogProgress bool + currentCycle SyncCycle // current cycle + stages []*Stage + revertOrder []*Stage + pruningOrder []*Stage + timings []Timing + logPrefixes []string + + evtDownloadFinished event.Feed // channel for each download task finished + evtDownloadFinishedSubscribed bool + evtDownloadStarted event.Feed // channel for each download has started + evtDownloadStartedSubscribed bool +} + +// BlockWithSig the serialization structure for request DownloaderRequest_BLOCKWITHSIG +// The block is encoded as block + commit signature +type BlockWithSig struct { + Block *types.Block + CommitSigAndBitmap []byte +} + +type Timing struct { + isRevert bool + isCleanUp bool + stage SyncStageID + took time.Duration +} + +type SyncCycle struct { + Number uint64 + TargetHeight uint64 + lock sync.RWMutex +} + +func (s *StagedStreamSync) Len() int { return len(s.stages) } +func (s *StagedStreamSync) Context() context.Context { return s.ctx } +func (s *StagedStreamSync) Blockchain() core.BlockChain { return s.bc } +func (s *StagedStreamSync) DB() kv.RwDB { return s.db } +func (s *StagedStreamSync) IsBeacon() bool { return s.isBeacon } +func (s *StagedStreamSync) IsExplorer() bool { return s.isExplorer } +func (s *StagedStreamSync) LogPrefix() string { + if s == nil { + return "" + } + return s.logPrefixes[s.currentStage] +} +func (s *StagedStreamSync) PrevRevertPoint() *uint64 { return s.prevRevertPoint } + +func (s *StagedStreamSync) NewRevertState(id SyncStageID, revertPoint uint64) *RevertState { + return &RevertState{id, revertPoint, s} +} + +func (s *StagedStreamSync) CleanUpStageState(id SyncStageID, forwardProgress uint64, tx kv.Tx, db kv.RwDB) (*CleanUpState, error) { + var pruneProgress uint64 + var err error + + if errV := CreateView(context.Background(), db, tx, func(tx kv.Tx) error { + pruneProgress, err = GetStageCleanUpProgress(tx, id, s.isBeacon) + if err != nil { + return err + } + return nil + }); errV != nil { + return nil, errV + } + + return &CleanUpState{id, forwardProgress, pruneProgress, s}, nil +} + +func (s *StagedStreamSync) NextStage() { + if s == nil { + return + } + s.currentStage++ +} + +// IsBefore returns true if stage1 goes before stage2 in staged sync +func (s *StagedStreamSync) IsBefore(stage1, stage2 SyncStageID) bool { + idx1 := -1 + idx2 := -1 + for i, stage := range s.stages { + if stage.ID == stage1 { + idx1 = i + } + + if stage.ID == stage2 { + idx2 = i + } + } + + return idx1 < idx2 +} + +// IsAfter returns true if stage1 goes after stage2 in staged sync +func (s *StagedStreamSync) IsAfter(stage1, stage2 SyncStageID) bool { + idx1 := -1 + idx2 := -1 + for i, stage := range s.stages { + if stage.ID == stage1 { + idx1 = i + } + + if stage.ID == stage2 { + idx2 = i + } + } + + return idx1 > idx2 +} + +func (s *StagedStreamSync) RevertTo(revertPoint uint64, invalidBlockNumber uint64, invalidBlockHash common.Hash, invalidBlockStreamID sttypes.StreamID) { + utils.Logger().Info(). + Interface("invalidBlockNumber", invalidBlockNumber). + Interface("invalidBlockHash", invalidBlockHash). + Interface("invalidBlockStreamID", invalidBlockStreamID). + Uint64("revertPoint", revertPoint). + Msgf(WrapStagedSyncMsg("Reverting blocks")) + s.revertPoint = &revertPoint + if invalidBlockNumber > 0 || invalidBlockHash != (common.Hash{}) { + resetBadStreams := !s.invalidBlock.Active + s.invalidBlock.set(invalidBlockNumber, invalidBlockHash, resetBadStreams) + s.invalidBlock.addBadStream(invalidBlockStreamID) + } +} + +func (s *StagedStreamSync) Done() { + s.currentStage = uint(len(s.stages)) + s.revertPoint = nil +} + +func (s *StagedStreamSync) IsDone() bool { + return s.currentStage >= uint(len(s.stages)) && s.revertPoint == nil +} + +func (s *StagedStreamSync) SetCurrentStage(id SyncStageID) error { + for i, stage := range s.stages { + if stage.ID == id { + s.currentStage = uint(i) + return nil + } + } + + return ErrStageNotFound +} + +func (s *StagedStreamSync) StageState(stage SyncStageID, tx kv.Tx, db kv.RwDB) (*StageState, error) { + var blockNum uint64 + var err error + if errV := CreateView(context.Background(), db, tx, func(rtx kv.Tx) error { + blockNum, err = GetStageProgress(rtx, stage, s.isBeacon) + if err != nil { + return err + } + return nil + }); errV != nil { + return nil, errV + } + + return &StageState{s, stage, blockNum}, nil +} + +func (s *StagedStreamSync) cleanUp(fromStage int, db kv.RwDB, tx kv.RwTx, firstCycle bool) error { + found := false + for i := 0; i < len(s.pruningOrder); i++ { + if s.pruningOrder[i].ID == s.stages[fromStage].ID { + found = true + } + if !found || s.pruningOrder[i] == nil || s.pruningOrder[i].Disabled { + continue + } + if err := s.pruneStage(firstCycle, s.pruningOrder[i], db, tx); err != nil { + panic(err) + } + } + return nil +} + +func New(ctx context.Context, + bc core.BlockChain, + db kv.RwDB, + stagesList []*Stage, + isBeacon bool, + protocol syncProtocol, + useMemDB bool, + config Config, + logger zerolog.Logger, +) *StagedStreamSync { + + revertStages := make([]*Stage, len(stagesList)) + for i, stageIndex := range DefaultRevertOrder { + for _, s := range stagesList { + if s.ID == stageIndex { + revertStages[i] = s + break + } + } + } + pruneStages := make([]*Stage, len(stagesList)) + for i, stageIndex := range DefaultCleanUpOrder { + for _, s := range stagesList { + if s.ID == stageIndex { + pruneStages[i] = s + break + } + } + } + + logPrefixes := make([]string, len(stagesList)) + for i := range stagesList { + logPrefixes[i] = fmt.Sprintf("%d/%d %s", i+1, len(stagesList), stagesList[i].ID) + } + + status := newStatus() + + return &StagedStreamSync{ + ctx: ctx, + bc: bc, + isBeacon: isBeacon, + db: db, + protocol: protocol, + gbm: nil, + status: status, + inserted: 0, + config: config, + logger: logger, + stages: stagesList, + currentStage: 0, + revertOrder: revertStages, + pruningOrder: pruneStages, + logPrefixes: logPrefixes, + UseMemDB: useMemDB, + } +} + +func (s *StagedStreamSync) doGetCurrentNumberRequest() (uint64, sttypes.StreamID, error) { + ctx, cancel := context.WithTimeout(s.ctx, 10*time.Second) + defer cancel() + + bn, stid, err := s.protocol.GetCurrentBlockNumber(ctx, syncproto.WithHighPriority()) + if err != nil { + return 0, stid, err + } + return bn, stid, nil +} + +func (s *StagedStreamSync) promLabels() prometheus.Labels { + sid := s.bc.ShardID() + return prometheus.Labels{"ShardID": fmt.Sprintf("%d", sid)} +} + +func (s *StagedStreamSync) checkHaveEnoughStreams() error { + numStreams := s.protocol.NumStreams() + if numStreams < s.config.MinStreams { + return fmt.Errorf("number of streams smaller than minimum: %v < %v", + numStreams, s.config.MinStreams) + } + return nil +} + +func (s *StagedStreamSync) SetNewContext(ctx context.Context) error { + for _, s := range s.stages { + s.Handler.SetStageContext(ctx) + } + return nil +} + +func (s *StagedStreamSync) Run(db kv.RwDB, tx kv.RwTx, firstCycle bool) error { + s.prevRevertPoint = nil + s.timings = s.timings[:0] + + for !s.IsDone() { + if s.revertPoint != nil { + s.prevRevertPoint = s.revertPoint + s.revertPoint = nil + if !s.invalidBlock.Active { + for j := 0; j < len(s.revertOrder); j++ { + if s.revertOrder[j] == nil || s.revertOrder[j].Disabled { + continue + } + if err := s.revertStage(firstCycle, s.revertOrder[j], db, tx); err != nil { + utils.Logger().Error(). + Err(err). + Interface("stage id", s.revertOrder[j].ID). + Msgf(WrapStagedSyncMsg("revert stage failed")) + return err + } + } + } + if err := s.SetCurrentStage(s.stages[0].ID); err != nil { + return err + } + firstCycle = false + } + + stage := s.stages[s.currentStage] + + if stage.Disabled { + utils.Logger().Trace(). + Msgf(WrapStagedSyncMsg(fmt.Sprintf("%s disabled. %s", stage.ID, stage.DisabledDescription))) + + s.NextStage() + continue + } + + if err := s.runStage(stage, db, tx, firstCycle, s.invalidBlock.Active); err != nil { + utils.Logger().Error(). + Err(err). + Interface("stage id", stage.ID). + Msgf(WrapStagedSyncMsg("stage failed")) + return err + } + s.NextStage() + } + + if err := s.cleanUp(0, db, tx, firstCycle); err != nil { + utils.Logger().Error(). + Err(err). + Msgf(WrapStagedSyncMsg("stages cleanup failed")) + return err + } + if err := s.SetCurrentStage(s.stages[0].ID); err != nil { + return err + } + if err := printLogs(tx, s.timings); err != nil { + return err + } + s.currentStage = 0 + return nil +} + +func CreateView(ctx context.Context, db kv.RwDB, tx kv.Tx, f func(tx kv.Tx) error) error { + if tx != nil { + return f(tx) + } + return db.View(context.Background(), func(etx kv.Tx) error { + return f(etx) + }) +} + +func ByteCount(b uint64) string { + const unit = 1024 + if b < unit { + return fmt.Sprintf("%dB", b) + } + div, exp := uint64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f%cB", + float64(b)/float64(div), "KMGTPE"[exp]) +} + +func printLogs(tx kv.RwTx, timings []Timing) error { + var logCtx []interface{} + count := 0 + for i := range timings { + if timings[i].took < 50*time.Millisecond { + continue + } + count++ + if count == 50 { + break + } + if timings[i].isRevert { + logCtx = append(logCtx, "Revert "+string(timings[i].stage), timings[i].took.Truncate(time.Millisecond).String()) + } else if timings[i].isCleanUp { + logCtx = append(logCtx, "CleanUp "+string(timings[i].stage), timings[i].took.Truncate(time.Millisecond).String()) + } else { + logCtx = append(logCtx, string(timings[i].stage), timings[i].took.Truncate(time.Millisecond).String()) + } + } + if len(logCtx) > 0 { + utils.Logger().Info(). + Msgf(WrapStagedSyncMsg(fmt.Sprintf("Timings (slower than 50ms) %v", logCtx...))) + } + + if tx == nil { + return nil + } + + if len(logCtx) > 0 { // also don't print this logs if everything is fast + buckets := Buckets + bucketSizes := make([]interface{}, 0, 2*len(buckets)) + for _, bucket := range buckets { + sz, err1 := tx.BucketSize(bucket) + if err1 != nil { + return err1 + } + bucketSizes = append(bucketSizes, bucket, ByteCount(sz)) + } + utils.Logger().Info(). + Msgf(WrapStagedSyncMsg(fmt.Sprintf("Tables %v", bucketSizes...))) + } + tx.CollectMetrics() + return nil +} + +func (s *StagedStreamSync) runStage(stage *Stage, db kv.RwDB, tx kv.RwTx, firstCycle bool, invalidBlockRevert bool) (err error) { + start := time.Now() + stageState, err := s.StageState(stage.ID, tx, db) + if err != nil { + return err + } + + if err = stage.Handler.Exec(firstCycle, invalidBlockRevert, stageState, s, tx); err != nil { + utils.Logger().Error(). + Err(err). + Interface("stage id", stage.ID). + Msgf(WrapStagedSyncMsg("stage failed")) + return fmt.Errorf("[%s] %w", s.LogPrefix(), err) + } + + took := time.Since(start) + if took > 60*time.Second { + logPrefix := s.LogPrefix() + utils.Logger().Info(). + Msgf(WrapStagedSyncMsg(fmt.Sprintf("%s: DONE in %d", logPrefix, took))) + + } + s.timings = append(s.timings, Timing{stage: stage.ID, took: took}) + return nil +} + +func (s *StagedStreamSync) revertStage(firstCycle bool, stage *Stage, db kv.RwDB, tx kv.RwTx) error { + start := time.Now() + stageState, err := s.StageState(stage.ID, tx, db) + if err != nil { + return err + } + + revert := s.NewRevertState(stage.ID, *s.revertPoint) + + if stageState.BlockNumber <= revert.RevertPoint { + return nil + } + + if err = s.SetCurrentStage(stage.ID); err != nil { + return err + } + + err = stage.Handler.Revert(firstCycle, revert, stageState, tx) + if err != nil { + return fmt.Errorf("[%s] %w", s.LogPrefix(), err) + } + + took := time.Since(start) + if took > 60*time.Second { + logPrefix := s.LogPrefix() + utils.Logger().Info(). + Msgf(WrapStagedSyncMsg(fmt.Sprintf("%s: Revert done in %d", logPrefix, took))) + } + s.timings = append(s.timings, Timing{isRevert: true, stage: stage.ID, took: took}) + return nil +} + +func (s *StagedStreamSync) pruneStage(firstCycle bool, stage *Stage, db kv.RwDB, tx kv.RwTx) error { + start := time.Now() + + stageState, err := s.StageState(stage.ID, tx, db) + if err != nil { + return err + } + + prune, err := s.CleanUpStageState(stage.ID, stageState.BlockNumber, tx, db) + if err != nil { + return err + } + if err = s.SetCurrentStage(stage.ID); err != nil { + return err + } + + err = stage.Handler.CleanUp(firstCycle, prune, tx) + if err != nil { + return fmt.Errorf("[%s] %w", s.LogPrefix(), err) + } + + took := time.Since(start) + if took > 60*time.Second { + logPrefix := s.LogPrefix() + utils.Logger().Info(). + Msgf(WrapStagedSyncMsg(fmt.Sprintf("%s: CleanUp done in %d", logPrefix, took))) + } + s.timings = append(s.timings, Timing{isCleanUp: true, stage: stage.ID, took: took}) + return nil +} + +// DisableAllStages disables all stages including their reverts +func (s *StagedStreamSync) DisableAllStages() []SyncStageID { + var backupEnabledIds []SyncStageID + for i := range s.stages { + if !s.stages[i].Disabled { + backupEnabledIds = append(backupEnabledIds, s.stages[i].ID) + } + } + for i := range s.stages { + s.stages[i].Disabled = true + } + return backupEnabledIds +} + +func (s *StagedStreamSync) DisableStages(ids ...SyncStageID) { + for i := range s.stages { + for _, id := range ids { + if s.stages[i].ID != id { + continue + } + s.stages[i].Disabled = true + } + } +} + +func (s *StagedStreamSync) EnableStages(ids ...SyncStageID) { + for i := range s.stages { + for _, id := range ids { + if s.stages[i].ID != id { + continue + } + s.stages[i].Disabled = false + } + } +} diff --git a/api/service/stagedstreamsync/stages.go b/api/service/stagedstreamsync/stages.go new file mode 100644 index 0000000000..dc607416b8 --- /dev/null +++ b/api/service/stagedstreamsync/stages.go @@ -0,0 +1,71 @@ +package stagedstreamsync + +import ( + "github.com/ledgerwatch/erigon-lib/kv" +) + +// SyncStageID represents the stages in the Mode.StagedSync mode +type SyncStageID string + +const ( + Heads SyncStageID = "Heads" // Heads are downloaded + ShortRange SyncStageID = "ShortRange" // short range + SyncEpoch SyncStageID = "SyncEpoch" // epoch sync + BlockBodies SyncStageID = "BlockBodies" // Block bodies are downloaded, TxHash and UncleHash are getting verified + States SyncStageID = "States" // will construct most recent state from downloaded blocks + Finish SyncStageID = "Finish" // Nominal stage after all other stages +) + +func GetStageName(stage string, isBeacon bool, prune bool) string { + name := stage + if isBeacon { + name = "beacon_" + name + } + if prune { + name = "prune_" + name + } + return name +} + +func GetStageID(stage SyncStageID, isBeacon bool, prune bool) []byte { + return []byte(GetStageName(string(stage), isBeacon, prune)) +} + +func GetBucketName(bucketName string, isBeacon bool) string { + name := bucketName + if isBeacon { + name = "Beacon" + name + } + return name +} + +// GetStageProgress retrieves saved progress of a given sync stage from the database +func GetStageProgress(db kv.Getter, stage SyncStageID, isBeacon bool) (uint64, error) { + stgID := GetStageID(stage, isBeacon, false) + v, err := db.GetOne(kv.SyncStageProgress, stgID) + if err != nil { + return 0, err + } + return unmarshalData(v) +} + +// SaveStageProgress saves progress of given sync stage +func SaveStageProgress(db kv.Putter, stage SyncStageID, isBeacon bool, progress uint64) error { + stgID := GetStageID(stage, isBeacon, false) + return db.Put(kv.SyncStageProgress, stgID, marshalData(progress)) +} + +// GetStageCleanUpProgress retrieves saved progress of given sync stage from the database +func GetStageCleanUpProgress(db kv.Getter, stage SyncStageID, isBeacon bool) (uint64, error) { + stgID := GetStageID(stage, isBeacon, true) + v, err := db.GetOne(kv.SyncStageProgress, stgID) + if err != nil { + return 0, err + } + return unmarshalData(v) +} + +func SaveStageCleanUpProgress(db kv.Putter, stage SyncStageID, isBeacon bool, progress uint64) error { + stgID := GetStageID(stage, isBeacon, true) + return db.Put(kv.SyncStageProgress, stgID, marshalData(progress)) +} diff --git a/api/service/stagedstreamsync/syncing.go b/api/service/stagedstreamsync/syncing.go new file mode 100644 index 0000000000..676eb3551e --- /dev/null +++ b/api/service/stagedstreamsync/syncing.go @@ -0,0 +1,320 @@ +package stagedstreamsync + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/internal/utils" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/harmony-one/harmony/shard" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/mdbx" + "github.com/ledgerwatch/erigon-lib/kv/memdb" + "github.com/ledgerwatch/log/v3" + "github.com/pkg/errors" + "github.com/rs/zerolog" +) + +const ( + BlocksBucket = "BlockBodies" + BlockSignaturesBucket = "BlockSignatures" + StageProgressBucket = "StageProgress" + + // cache db keys + LastBlockHeight = "LastBlockHeight" + LastBlockHash = "LastBlockHash" +) + +var Buckets = []string{ + BlocksBucket, + BlockSignaturesBucket, + StageProgressBucket, +} + +// CreateStagedSync creates an instance of staged sync +func CreateStagedSync(ctx context.Context, + bc core.BlockChain, + UseMemDB bool, + protocol syncProtocol, + config Config, + logger zerolog.Logger, + logProgress bool, +) (*StagedStreamSync, error) { + + isBeacon := bc.ShardID() == shard.BeaconChainShardID + + var mainDB kv.RwDB + dbs := make([]kv.RwDB, config.Concurrency) + if UseMemDB { + mainDB = memdb.New() + for i := 0; i < config.Concurrency; i++ { + dbs[i] = memdb.New() + } + } else { + mainDB = mdbx.NewMDBX(log.New()).Path(GetBlockDbPath(isBeacon, -1)).MustOpen() + for i := 0; i < config.Concurrency; i++ { + dbPath := GetBlockDbPath(isBeacon, i) + dbs[i] = mdbx.NewMDBX(log.New()).Path(dbPath).MustOpen() + } + } + + if errInitDB := initDB(ctx, mainDB, dbs, config.Concurrency); errInitDB != nil { + return nil, errInitDB + } + + stageHeadsCfg := NewStageHeadersCfg(ctx, bc, mainDB) + stageShortRangeCfg := NewStageShortRangeCfg(ctx, bc, mainDB) + stageSyncEpochCfg := NewStageEpochCfg(ctx, bc, mainDB) + stageBodiesCfg := NewStageBodiesCfg(ctx, bc, mainDB, dbs, config.Concurrency, protocol, isBeacon, logProgress) + stageStatesCfg := NewStageStatesCfg(ctx, bc, mainDB, dbs, config.Concurrency, logger, logProgress) + stageFinishCfg := NewStageFinishCfg(ctx, mainDB) + + stages := DefaultStages(ctx, + stageHeadsCfg, + stageSyncEpochCfg, + stageShortRangeCfg, + stageBodiesCfg, + stageStatesCfg, + stageFinishCfg, + ) + + return New(ctx, + bc, + mainDB, + stages, + isBeacon, + protocol, + UseMemDB, + config, + logger, + ), nil +} + +// initDB inits the sync loop main database and create buckets +func initDB(ctx context.Context, mainDB kv.RwDB, dbs []kv.RwDB, concurrency int) error { + + // create buckets for mainDB + tx, errRW := mainDB.BeginRw(ctx) + if errRW != nil { + return errRW + } + defer tx.Rollback() + + for _, name := range Buckets { + if err := tx.CreateBucket(GetStageName(name, false, false)); err != nil { + return err + } + } + if err := tx.Commit(); err != nil { + return err + } + + // create buckets for block cache DBs + for _, db := range dbs { + tx, errRW := db.BeginRw(ctx) + if errRW != nil { + return errRW + } + + if err := tx.CreateBucket(BlocksBucket); err != nil { + return err + } + if err := tx.CreateBucket(BlockSignaturesBucket); err != nil { + return err + } + + if err := tx.Commit(); err != nil { + return err + } + } + + return nil +} + +func GetBlockDbPath(beacon bool, loopID int) string { + if beacon { + if loopID >= 0 { + return fmt.Sprintf("%s_%d", "cache/beacon_blocks_db", loopID) + } else { + return "cache/beacon_blocks_db_main" + } + } else { + if loopID >= 0 { + return fmt.Sprintf("%s_%d", "cache/blocks_db", loopID) + } else { + return "cache/blocks_db_main" + } + } +} + +// doSync does the long range sync. +// One LongRangeSync consists of several iterations. +// For each iteration, estimate the current block number, then fetch block & insert to blockchain +func (s *StagedStreamSync) doSync(downloaderContext context.Context, initSync bool) (int, error) { + + var totalInserted int + + s.initSync = initSync + + if err := s.checkPrerequisites(); err != nil { + return 0, err + } + + var estimatedHeight uint64 + if initSync { + if h, err := s.estimateCurrentNumber(); err != nil { + return 0, err + } else { + estimatedHeight = h + //TODO: use directly currentCycle var + s.status.setTargetBN(estimatedHeight) + } + if curBN := s.bc.CurrentBlock().NumberU64(); estimatedHeight <= curBN { + s.logger.Info().Uint64("current number", curBN).Uint64("target number", estimatedHeight). + Msg(WrapStagedSyncMsg("early return of long range sync")) + return 0, nil + } + + s.startSyncing() + defer s.finishSyncing() + } + + for { + ctx, cancel := context.WithCancel(downloaderContext) + s.ctx = ctx + s.SetNewContext(ctx) + + n, err := s.doSyncCycle(ctx, initSync) + if err != nil { + pl := s.promLabels() + pl["error"] = err.Error() + numFailedDownloadCounterVec.With(pl).Inc() + + cancel() + return totalInserted + n, err + } + cancel() + + totalInserted += n + + // if it's not long range sync, skip loop + if n < LastMileBlocksThreshold || !initSync { + return totalInserted, nil + } + } + +} + +func (s *StagedStreamSync) doSyncCycle(ctx context.Context, initSync bool) (int, error) { + + // TODO: initSync=true means currentCycleNumber==0, so we can remove initSync + + var totalInserted int + + s.inserted = 0 + startHead := s.bc.CurrentBlock().NumberU64() + canRunCycleInOneTransaction := false + + var tx kv.RwTx + if canRunCycleInOneTransaction { + var err error + if tx, err = s.DB().BeginRw(context.Background()); err != nil { + return totalInserted, err + } + defer tx.Rollback() + } + + startTime := time.Now() + + // Do one cycle of staged sync + initialCycle := s.currentCycle.Number == 0 + if err := s.Run(s.DB(), tx, initialCycle); err != nil { + utils.Logger().Error(). + Err(err). + Bool("isBeacon", s.isBeacon). + Uint32("shard", s.bc.ShardID()). + Uint64("currentHeight", startHead). + Msgf(WrapStagedSyncMsg("sync cycle failed")) + return totalInserted, err + } + + totalInserted += s.inserted + + s.currentCycle.lock.Lock() + s.currentCycle.Number++ + s.currentCycle.lock.Unlock() + + // calculating sync speed (blocks/second) + if s.LogProgress && s.inserted > 0 { + dt := time.Now().Sub(startTime).Seconds() + speed := float64(0) + if dt > 0 { + speed = float64(s.inserted) / dt + } + syncSpeed := fmt.Sprintf("%.2f", speed) + fmt.Println("sync speed:", syncSpeed, "blocks/s") + } + + return totalInserted, nil +} + +func (s *StagedStreamSync) startSyncing() { + s.status.startSyncing() + if s.evtDownloadStartedSubscribed { + s.evtDownloadStarted.Send(struct{}{}) + } +} + +func (s *StagedStreamSync) finishSyncing() { + s.status.finishSyncing() + if s.evtDownloadFinishedSubscribed { + s.evtDownloadFinished.Send(struct{}{}) + } +} + +func (s *StagedStreamSync) checkPrerequisites() error { + return s.checkHaveEnoughStreams() +} + +// estimateCurrentNumber roughly estimates the current block number. +// The block number does not need to be exact, but just a temporary target of the iteration +func (s *StagedStreamSync) estimateCurrentNumber() (uint64, error) { + var ( + cnResults = make(map[sttypes.StreamID]uint64) + lock sync.Mutex + wg sync.WaitGroup + ) + wg.Add(s.config.Concurrency) + for i := 0; i != s.config.Concurrency; i++ { + go func() { + defer wg.Done() + bn, stid, err := s.doGetCurrentNumberRequest() + if err != nil { + s.logger.Err(err).Str("streamID", string(stid)). + Msg(WrapStagedSyncMsg("getCurrentNumber request failed")) + if !errors.Is(err, context.Canceled) { + s.protocol.StreamFailed(stid, "getCurrentNumber request failed") + } + return + } + lock.Lock() + cnResults[stid] = bn + lock.Unlock() + }() + } + wg.Wait() + + if len(cnResults) == 0 { + select { + case <-s.ctx.Done(): + return 0, s.ctx.Err() + default: + } + return 0, errors.New("zero block number response from remote nodes") + } + bn := computeBlockNumberByMaxVote(cnResults) + return bn, nil +} diff --git a/api/service/stagedstreamsync/types.go b/api/service/stagedstreamsync/types.go new file mode 100644 index 0000000000..00cfba79a2 --- /dev/null +++ b/api/service/stagedstreamsync/types.go @@ -0,0 +1,287 @@ +package stagedstreamsync + +import ( + "container/heap" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/harmony/core/types" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" +) + +var ( + emptyHash common.Hash +) + +type status struct { + isSyncing bool + targetBN uint64 + lock sync.Mutex +} + +func newStatus() status { + return status{} +} + +func (s *status) startSyncing() { + s.lock.Lock() + defer s.lock.Unlock() + + s.isSyncing = true +} + +func (s *status) setTargetBN(val uint64) { + s.lock.Lock() + defer s.lock.Unlock() + + s.targetBN = val +} + +func (s *status) finishSyncing() { + s.lock.Lock() + defer s.lock.Unlock() + + s.isSyncing = false + s.targetBN = 0 +} + +func (s *status) get() (bool, uint64) { + s.lock.Lock() + defer s.lock.Unlock() + + return s.isSyncing, s.targetBN +} + +type getBlocksResult struct { + bns []uint64 + blocks []*types.Block + stid sttypes.StreamID +} + +type resultQueue struct { + results *priorityQueue + lock sync.Mutex +} + +func newResultQueue() *resultQueue { + pq := make(priorityQueue, 0, 200) // 200 - rough estimate + heap.Init(&pq) + return &resultQueue{ + results: &pq, + } +} + +// addBlockResults adds the blocks to the result queue to be processed by insertChainLoop. +// If a nil block is detected in the block list, will not process further blocks. +func (rq *resultQueue) addBlockResults(blocks []*types.Block, stid sttypes.StreamID) { + rq.lock.Lock() + defer rq.lock.Unlock() + + for _, block := range blocks { + if block == nil { + continue + } + heap.Push(rq.results, &blockResult{ + block: block, + stid: stid, + }) + } + return +} + +// popBlockResults pop a continuous list of blocks starting at expStartBN with capped size. +// Return the stale block numbers as the second return value +func (rq *resultQueue) popBlockResults(expStartBN uint64, cap int) ([]*blockResult, []uint64) { + rq.lock.Lock() + defer rq.lock.Unlock() + + var ( + res = make([]*blockResult, 0, cap) + stales []uint64 + ) + + for cnt := 0; rq.results.Len() > 0 && cnt < cap; cnt++ { + br := heap.Pop(rq.results).(*blockResult) + // stale block number + if br.block.NumberU64() < expStartBN { + stales = append(stales, br.block.NumberU64()) + continue + } + if br.block.NumberU64() != expStartBN { + heap.Push(rq.results, br) + return res, stales + } + res = append(res, br) + expStartBN++ + } + return res, stales +} + +// removeResultsByStreamID removes the block results of the given stream, returns the block +// number removed from the queue +func (rq *resultQueue) removeResultsByStreamID(stid sttypes.StreamID) []uint64 { + rq.lock.Lock() + defer rq.lock.Unlock() + + var removed []uint64 + +Loop: + for { + for i, res := range *rq.results { + blockRes := res.(*blockResult) + if blockRes.stid == stid { + rq.removeByIndex(i) + removed = append(removed, blockRes.block.NumberU64()) + goto Loop + } + } + break + } + return removed +} + +func (rq *resultQueue) length() int { + return len(*rq.results) +} + +func (rq *resultQueue) removeByIndex(index int) { + heap.Remove(rq.results, index) +} + +// bnPrioritizedItem is the item which uses block number to determine its priority +type bnPrioritizedItem interface { + getBlockNumber() uint64 +} + +type blockResult struct { + block *types.Block + stid sttypes.StreamID +} + +func (br *blockResult) getBlockNumber() uint64 { + return br.block.NumberU64() +} + +func blockResultsToBlocks(results []*blockResult) []*types.Block { + blocks := make([]*types.Block, 0, len(results)) + + for _, result := range results { + blocks = append(blocks, result.block) + } + return blocks +} + +type ( + prioritizedNumber uint64 + + prioritizedNumbers struct { + q *priorityQueue + } +) + +func (b prioritizedNumber) getBlockNumber() uint64 { + return uint64(b) +} + +func newPrioritizedNumbers() *prioritizedNumbers { + pqs := make(priorityQueue, 0) + heap.Init(&pqs) + return &prioritizedNumbers{ + q: &pqs, + } +} + +func (pbs *prioritizedNumbers) push(bn uint64) { + heap.Push(pbs.q, prioritizedNumber(bn)) +} + +func (pbs *prioritizedNumbers) pop() uint64 { + if pbs.q.Len() == 0 { + return 0 + } + item := heap.Pop(pbs.q) + return uint64(item.(prioritizedNumber)) +} + +func (pbs *prioritizedNumbers) length() int { + return len(*pbs.q) +} + +type ( + blockByNumber types.Block + + // blocksByNumber is the priority queue ordered by number + blocksByNumber struct { + q *priorityQueue + cap int + } +) + +func (b *blockByNumber) getBlockNumber() uint64 { + raw := (*types.Block)(b) + return raw.NumberU64() +} + +func newBlocksByNumber(cap int) *blocksByNumber { + pqs := make(priorityQueue, 0) + heap.Init(&pqs) + return &blocksByNumber{ + q: &pqs, + cap: cap, + } +} + +func (bs *blocksByNumber) push(b *types.Block) { + heap.Push(bs.q, (*blockByNumber)(b)) + for bs.q.Len() > bs.cap { + heap.Pop(bs.q) + } +} + +func (bs *blocksByNumber) pop() *types.Block { + if bs.q.Len() == 0 { + return nil + } + item := heap.Pop(bs.q) + return (*types.Block)(item.(*blockByNumber)) +} + +func (bs *blocksByNumber) len() int { + return bs.q.Len() +} + +// priorityQueue is a priority queue with lowest block number with highest priority +type priorityQueue []bnPrioritizedItem + +func (q priorityQueue) Len() int { + return len(q) +} + +func (q priorityQueue) Less(i, j int) bool { + bn1 := q[i].getBlockNumber() + bn2 := q[j].getBlockNumber() + return bn1 < bn2 // small block number has higher priority +} + +func (q priorityQueue) Swap(i, j int) { + q[i], q[j] = q[j], q[i] +} + +func (q *priorityQueue) Push(x interface{}) { + item, ok := x.(bnPrioritizedItem) + if !ok { + panic("wrong type of getBlockNumber interface") + } + *q = append(*q, item) +} + +func (q *priorityQueue) Pop() interface{} { + prev := *q + n := len(prev) + if n == 0 { + return nil + } + res := prev[n-1] + *q = prev[0 : n-1] + return res +} diff --git a/api/service/stagedstreamsync/types_test.go b/api/service/stagedstreamsync/types_test.go new file mode 100644 index 0000000000..1890608b25 --- /dev/null +++ b/api/service/stagedstreamsync/types_test.go @@ -0,0 +1,266 @@ +package stagedstreamsync + +import ( + "container/heap" + "fmt" + "math/big" + "strings" + "testing" + + "github.com/harmony-one/harmony/block" + headerV3 "github.com/harmony-one/harmony/block/v3" + "github.com/harmony-one/harmony/core/types" + bls_cosi "github.com/harmony-one/harmony/crypto/bls" + sttypes "github.com/harmony-one/harmony/p2p/stream/types" +) + +func TestResultQueue_AddBlockResults(t *testing.T) { + tests := []struct { + initBNs []uint64 + addBNs []uint64 + expSize int + }{ + { + initBNs: []uint64{}, + addBNs: []uint64{1, 2, 3, 4}, + expSize: 4, + }, + { + initBNs: []uint64{1, 2, 3, 4}, + addBNs: []uint64{5, 6, 7, 8}, + expSize: 8, + }, + } + for i, test := range tests { + rq := makeTestResultQueue(test.initBNs) + rq.addBlockResults(makeTestBlocks(test.addBNs), "") + + if rq.results.Len() != test.expSize { + t.Errorf("Test %v: unexpected size: %v / %v", i, rq.results.Len(), test.expSize) + } + } +} + +func TestResultQueue_PopBlockResults(t *testing.T) { + tests := []struct { + initBNs []uint64 + cap int + expStart uint64 + expSize int + staleSize int + }{ + { + initBNs: []uint64{1, 2, 3, 4, 5}, + cap: 3, + expStart: 1, + expSize: 3, + staleSize: 0, + }, + { + initBNs: []uint64{1, 2, 3, 4, 5}, + cap: 10, + expStart: 1, + expSize: 5, + staleSize: 0, + }, + { + initBNs: []uint64{1, 3, 4, 5}, + cap: 10, + expStart: 1, + expSize: 1, + staleSize: 0, + }, + { + initBNs: []uint64{1, 2, 3, 4, 5}, + cap: 10, + expStart: 0, + expSize: 0, + staleSize: 0, + }, + { + initBNs: []uint64{1, 1, 1, 1, 2}, + cap: 10, + expStart: 1, + expSize: 2, + staleSize: 3, + }, + { + initBNs: []uint64{1, 2, 3, 4, 5}, + cap: 10, + expStart: 2, + expSize: 4, + staleSize: 1, + }, + } + for i, test := range tests { + rq := makeTestResultQueue(test.initBNs) + res, stales := rq.popBlockResults(test.expStart, test.cap) + if len(res) != test.expSize { + t.Errorf("Test %v: unexpect size %v / %v", i, len(res), test.expSize) + } + if len(stales) != test.staleSize { + t.Errorf("Test %v: unexpect stale size %v / %v", i, len(stales), test.staleSize) + } + } +} + +func TestResultQueue_RemoveResultsByStreamID(t *testing.T) { + tests := []struct { + rq *resultQueue + rmStreamID sttypes.StreamID + removed int + expSize int + }{ + { + rq: makeTestResultQueue([]uint64{1, 2, 3, 4}), + rmStreamID: "test stream id", + removed: 4, + expSize: 0, + }, + { + rq: func() *resultQueue { + rq := makeTestResultQueue([]uint64{2, 3, 4, 5}) + rq.addBlockResults([]*types.Block{ + makeTestBlock(1), + makeTestBlock(5), + makeTestBlock(6), + }, "another test stream id") + return rq + }(), + rmStreamID: "test stream id", + removed: 4, + expSize: 3, + }, + { + rq: func() *resultQueue { + rq := makeTestResultQueue([]uint64{2, 3, 4, 5}) + rq.addBlockResults([]*types.Block{ + makeTestBlock(1), + makeTestBlock(5), + makeTestBlock(6), + }, "another test stream id") + return rq + }(), + rmStreamID: "another test stream id", + removed: 3, + expSize: 4, + }, + } + for i, test := range tests { + res := test.rq.removeResultsByStreamID(test.rmStreamID) + if len(res) != test.removed { + t.Errorf("Test %v: unexpected number removed %v / %v", i, len(res), test.removed) + } + if gotSize := test.rq.results.Len(); gotSize != test.expSize { + t.Errorf("Test %v: unexpected number after removal %v / %v", i, gotSize, test.expSize) + } + } +} + +func makeTestResultQueue(bns []uint64) *resultQueue { + rq := newResultQueue() + for _, bn := range bns { + heap.Push(rq.results, &blockResult{ + block: makeTestBlock(bn), + stid: "test stream id", + }) + } + return rq +} + +func TestPrioritizedBlocks(t *testing.T) { + addBNs := []uint64{4, 7, 6, 9} + + bns := newPrioritizedNumbers() + for _, bn := range addBNs { + bns.push(bn) + } + prevBN := uint64(0) + for len(*bns.q) > 0 { + b := bns.pop() + if b < prevBN { + t.Errorf("number not incrementing") + } + prevBN = b + } + if last := bns.pop(); last != 0 { + t.Errorf("last elem is not 0") + } +} + +func TestBlocksByNumber(t *testing.T) { + addBNs := []uint64{4, 7, 6, 9} + + bns := newBlocksByNumber(10) + for _, bn := range addBNs { + bns.push(makeTestBlock(bn)) + } + if bns.len() != len(addBNs) { + t.Errorf("size unexpected: %v / %v", bns.len(), len(addBNs)) + } + prevBN := uint64(0) + for len(*bns.q) > 0 { + b := bns.pop() + if b.NumberU64() < prevBN { + t.Errorf("number not incrementing") + } + prevBN = b.NumberU64() + } + if lastBlock := bns.pop(); lastBlock != nil { + t.Errorf("last block is not nil") + } +} + +func TestPriorityQueue(t *testing.T) { + testBNs := []uint64{1, 9, 2, 4, 5, 12} + pq := make(priorityQueue, 0, 10) + heap.Init(&pq) + for _, bn := range testBNs { + heap.Push(&pq, &blockResult{ + block: makeTestBlock(bn), + stid: "", + }) + } + cmpBN := uint64(0) + for pq.Len() > 0 { + bn := heap.Pop(&pq).(*blockResult).block.NumberU64() + if bn < cmpBN { + t.Errorf("not incrementing") + } + cmpBN = bn + } + if pq.Len() != 0 { + t.Errorf("after poping, size not 0") + } +} + +func makeTestBlocks(bns []uint64) []*types.Block { + blocks := make([]*types.Block, 0, len(bns)) + for _, bn := range bns { + blocks = append(blocks, makeTestBlock(bn)) + } + return blocks +} + +func makeTestBlock(bn uint64) *types.Block { + testHeader := &block.Header{Header: headerV3.NewHeader()} + testHeader.SetNumber(big.NewInt(int64(bn))) + testHeader.SetLastCommitSignature(bls_cosi.SerializedSignature{}) + testHeader.SetLastCommitBitmap(make([]byte, 10)) + block := types.NewBlockWithHeader(testHeader) + block.SetCurrentCommitSig(make([]byte, 106)) + return block +} + +func assertError(got, expect error) error { + if (got == nil) != (expect == nil) { + return fmt.Errorf("unexpected error [%v] / [%v]", got, expect) + } + if (got == nil) || (expect == nil) { + return nil + } + if !strings.Contains(got.Error(), expect.Error()) { + return fmt.Errorf("unexpected error [%v] / [%v]", got, expect) + } + return nil +} diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 437a91dc59..6c83636673 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -39,6 +39,7 @@ import ( "github.com/harmony-one/harmony/api/service" "github.com/harmony-one/harmony/api/service/pprof" "github.com/harmony-one/harmony/api/service/prometheus" + "github.com/harmony-one/harmony/api/service/stagedstreamsync" "github.com/harmony-one/harmony/api/service/synchronize" "github.com/harmony-one/harmony/common/fdlimit" "github.com/harmony-one/harmony/common/ntp" @@ -415,7 +416,11 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { // Setup services if hc.Sync.Enabled { - setupSyncService(currentNode, myHost, hc) + if hc.Sync.StagedSync { + setupStagedSyncService(currentNode, myHost, hc) + } else { + setupSyncService(currentNode, myHost, hc) + } } if currentNode.NodeConfig.Role() == nodeconfig.Validator { currentNode.RegisterValidatorServices() @@ -906,6 +911,45 @@ func setupSyncService(node *node.Node, host p2p.Host, hc harmonyconfig.HarmonyCo } } +func setupStagedSyncService(node *node.Node, host p2p.Host, hc harmonyconfig.HarmonyConfig) { + blockchains := []core.BlockChain{node.Blockchain()} + if !node.IsRunningBeaconChain() { + blockchains = append(blockchains, node.Beaconchain()) + } + + sConfig := stagedstreamsync.Config{ + ServerOnly: !hc.Sync.Downloader, + Network: nodeconfig.NetworkType(hc.Network.NetworkType), + Concurrency: hc.Sync.Concurrency, + MinStreams: hc.Sync.MinPeers, + InitStreams: hc.Sync.InitStreams, + SmSoftLowCap: hc.Sync.DiscSoftLowCap, + SmHardLowCap: hc.Sync.DiscHardLowCap, + SmHiCap: hc.Sync.DiscHighCap, + SmDiscBatch: hc.Sync.DiscBatch, + LogProgress: node.NodeConfig.LogProgress, + } + + // If we are running side chain, we will need to do some extra works for beacon + // sync. + if !node.IsRunningBeaconChain() { + sConfig.BHConfig = &stagedstreamsync.BeaconHelperConfig{ + BlockC: node.BeaconBlockChannel, + InsertHook: node.BeaconSyncHook, + } + } + + //Setup stream sync service + s := stagedstreamsync.NewService(host, blockchains, sConfig) + + node.RegisterService(service.StagedStreamSync, s) + + d := s.Downloaders.GetShardDownloader(node.Blockchain().ShardID()) + if hc.Sync.Downloader && hc.General.NodeType != nodeTypeExplorer { + node.Consensus.SetDownloader(d) // Set downloader when stream client is active + } +} + func setupBlacklist(hc harmonyconfig.HarmonyConfig) (map[ethCommon.Address]struct{}, error) { rosetta_common.InitRosettaFile(hc.TxPool.RosettaFixFile) diff --git a/go.mod b/go.mod index 901fc1f0e3..f70708b562 100644 --- a/go.mod +++ b/go.mod @@ -165,6 +165,7 @@ require ( github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect + github.com/libp2p/go-libp2p-core v0.20.1 // indirect github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect github.com/libp2p/go-msgio v0.2.0 // indirect diff --git a/go.sum b/go.sum index 0b7ab4b961..19d3d9e3da 100644 --- a/go.sum +++ b/go.sum @@ -594,6 +594,8 @@ github.com/libp2p/go-libp2p v0.24.0 h1:DQk/5bBon+yUVIGTeRVBmOYpZzoBHx/VTC0xoLgJG github.com/libp2p/go-libp2p v0.24.0/go.mod h1:28t24CYDlnBs23rIs1OclU89YbhgibrBq2LFbMe+cFw= github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= +github.com/libp2p/go-libp2p-core v0.20.1 h1:fQz4BJyIFmSZAiTbKV8qoYhEH5Dtv/cVhZbG3Ib/+Cw= +github.com/libp2p/go-libp2p-core v0.20.1/go.mod h1:6zR8H7CvQWgYLsbG4on6oLNSGcyKaYFSEYyDt51+bIY= github.com/libp2p/go-libp2p-kad-dht v0.19.0 h1:2HuiInHZTm9ZvQajaqdaPLHr0PCKKigWiflakimttE0= github.com/libp2p/go-libp2p-kad-dht v0.19.0/go.mod h1:qPIXdiZsLczhV4/+4EO1jE8ae0YCW4ZOogc4WVIyTEU= github.com/libp2p/go-libp2p-kbucket v0.5.0 h1:g/7tVm8ACHDxH29BGrpsQlnNeu+6OF1A9bno/4/U1oA= diff --git a/hmy/downloader/const.go b/hmy/downloader/const.go index ac292d9747..a6cafb918b 100644 --- a/hmy/downloader/const.go +++ b/hmy/downloader/const.go @@ -35,7 +35,8 @@ type ( // Only run stream sync protocol as a server. // TODO: remove this when stream sync is fully up. ServerOnly bool - + // use staged sync + Staged bool // parameters Network nodeconfig.NetworkType Concurrency int // Number of concurrent sync requests diff --git a/hmy/downloader/metric.go b/hmy/downloader/metric.go index 2995db7b31..9338fd1304 100644 --- a/hmy/downloader/metric.go +++ b/hmy/downloader/metric.go @@ -23,7 +23,7 @@ var ( consensusTriggeredDownloadCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "consensus_trigger", Help: "number of times consensus triggered download task", }, @@ -33,7 +33,7 @@ var ( longRangeSyncedBlockCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "num_blocks_synced_long_range", Help: "number of blocks synced in long range sync", }, @@ -43,7 +43,7 @@ var ( longRangeFailInsertedBlockCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "num_blocks_failed_long_range", Help: "number of blocks failed to insert into change in long range sync", }, @@ -53,7 +53,7 @@ var ( numShortRangeCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "num_short_range", Help: "number of short range sync is triggered", }, @@ -63,7 +63,7 @@ var ( numFailedDownloadCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "failed_download", Help: "number of downloading is failed", }, @@ -73,7 +73,7 @@ var ( numBlocksInsertedShortRangeHistogramVec = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "num_blocks_inserted_short_range", Help: "number of blocks inserted for each short range sync", // Buckets: 0, 1, 2, 4, +INF (capped at 10) @@ -85,7 +85,7 @@ var ( numBlocksInsertedBeaconHelperCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "num_blocks_inserted_beacon_helper", Help: "number of blocks inserted from beacon helper", }, diff --git a/hmy/downloader/shortrange.go b/hmy/downloader/shortrange.go index f1f6e402ce..8276911d4f 100644 --- a/hmy/downloader/shortrange.go +++ b/hmy/downloader/shortrange.go @@ -20,7 +20,7 @@ import ( // doShortRangeSync does the short range sync. // Compared with long range sync, short range sync is more focused on syncing to the latest block. // It consist of 3 steps: -// 1. Obtain the block hashes and ompute the longest hash chain.. +// 1. Obtain the block hashes and compute the longest hash chain.. // 2. Get blocks by hashes from computed hash chain. // 3. Insert the blocks to blockchain. func (d *Downloader) doShortRangeSync() (int, error) { diff --git a/node/node_syncing.go b/node/node_syncing.go index d9a6c95b48..2219be96da 100644 --- a/node/node_syncing.go +++ b/node/node_syncing.go @@ -22,11 +22,11 @@ import ( "github.com/harmony-one/harmony/api/service/legacysync" legdownloader "github.com/harmony-one/harmony/api/service/legacysync/downloader" downloader_pb "github.com/harmony-one/harmony/api/service/legacysync/downloader/proto" + "github.com/harmony-one/harmony/api/service/stagedstreamsync" "github.com/harmony-one/harmony/api/service/stagedsync" "github.com/harmony-one/harmony/api/service/synchronize" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" - "github.com/harmony-one/harmony/hmy/downloader" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/node/worker" @@ -811,7 +811,7 @@ func (node *Node) legacySyncStatus(shardID uint32) (bool, uint64, uint64) { } } -// IsOutOfSync return whether the node is out of sync of the given hsardID +// IsOutOfSync return whether the node is out of sync of the given shardID func (node *Node) IsOutOfSync(shardID uint32) bool { ds := node.getDownloaders() if ds == nil || !ds.IsActive() { @@ -859,14 +859,36 @@ func (node *Node) SyncPeers() map[string]int { return res } -func (node *Node) getDownloaders() *downloader.Downloaders { - syncService := node.serviceManager.GetService(service.Synchronize) - if syncService == nil { - return nil - } - dsService, ok := syncService.(*synchronize.Service) - if !ok { - return nil +type Downloaders interface { + Start() + Close() + DownloadAsync(shardID uint32) + // GetShardDownloader(shardID uint32) *Downloader + NumPeers() map[uint32]int + SyncStatus(shardID uint32) (bool, uint64, uint64) + IsActive() bool +} + +func (node *Node) getDownloaders() Downloaders { + if node.NodeConfig.StagedSync { + syncService := node.serviceManager.GetService(service.StagedStreamSync) + if syncService == nil { + return nil + } + dsService, ok := syncService.(*stagedstreamsync.StagedStreamSyncService) + if !ok { + return nil + } + return dsService.Downloaders + } else { + syncService := node.serviceManager.GetService(service.Synchronize) + if syncService == nil { + return nil + } + dsService, ok := syncService.(*synchronize.Service) + if !ok { + return nil + } + return dsService.Downloaders } - return dsService.Downloaders } diff --git a/p2p/stream/common/requestmanager/interface_test.go b/p2p/stream/common/requestmanager/interface_test.go index fe163164d7..76399e859e 100644 --- a/p2p/stream/common/requestmanager/interface_test.go +++ b/p2p/stream/common/requestmanager/interface_test.go @@ -114,6 +114,18 @@ func (st *testStream) CloseOnExit() error { return nil } +func (st *testStream) FailedTimes() int { + return 0 +} + +func (st *testStream) AddFailedTimes() { + return +} + +func (st *testStream) ResetFailedRimes() { + return +} + func makeDummyTestStreams(indexes []int) []sttypes.Stream { sts := make([]sttypes.Stream, 0, len(indexes)) diff --git a/p2p/stream/common/streammanager/interface_test.go b/p2p/stream/common/streammanager/interface_test.go index fc280b47ed..e308c3abab 100644 --- a/p2p/stream/common/streammanager/interface_test.go +++ b/p2p/stream/common/streammanager/interface_test.go @@ -70,6 +70,18 @@ func (st *testStream) ReadBytes() ([]byte, error) { return nil, nil } +func (st *testStream) FailedTimes() int { + return 0 +} + +func (st *testStream) AddFailedTimes() { + return +} + +func (st *testStream) ResetFailedRimes() { + return +} + func (st *testStream) Close() error { if st.closed { return errors.New("already closed") diff --git a/p2p/stream/common/streammanager/streammanager.go b/p2p/stream/common/streammanager/streammanager.go index 4b3d19b921..33270e1c7b 100644 --- a/p2p/stream/common/streammanager/streammanager.go +++ b/p2p/stream/common/streammanager/streammanager.go @@ -10,6 +10,7 @@ import ( "github.com/harmony-one/abool" "github.com/harmony-one/harmony/internal/utils" sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/harmony-one/harmony/shard" "github.com/libp2p/go-libp2p/core/network" libp2p_peer "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" @@ -73,6 +74,9 @@ func newStreamManager(pid sttypes.ProtoID, host host, pf peerFinder, handleStrea protoSpec, _ := sttypes.ProtoIDToProtoSpec(pid) + fmt.Println("my peer id: ", host.ID().String()) + fmt.Println("my proto id: ", pid) + return &streamManager{ myProtoID: pid, myProtoSpec: protoSpec, @@ -234,6 +238,9 @@ func (sm *streamManager) sanityCheckStream(st sttypes.Stream) error { if mySpec.ShardID != rmSpec.ShardID { return fmt.Errorf("unexpected shard ID: %v/%v", rmSpec.ShardID, mySpec.ShardID) } + if mySpec.ShardID == shard.BeaconChainShardID && !rmSpec.BeaconNode { + return fmt.Errorf("unexpected beacon node with shard ID: %v/%v", rmSpec.ShardID, mySpec.ShardID) + } return nil } @@ -323,7 +330,7 @@ func (sm *streamManager) discoverAndSetupStream(discCtx context.Context) (int, e } func (sm *streamManager) discover(ctx context.Context) (<-chan libp2p_peer.AddrInfo, error) { - protoID := string(sm.myProtoID) + protoID := sm.targetProtoID() discBatch := sm.config.DiscBatch if sm.config.HiCap-sm.streams.size() < sm.config.DiscBatch { discBatch = sm.config.HiCap - sm.streams.size() @@ -340,6 +347,14 @@ func (sm *streamManager) discover(ctx context.Context) (<-chan libp2p_peer.AddrI return sm.pf.FindPeers(ctx2, protoID, discBatch) } +func (sm *streamManager) targetProtoID() string { + targetSpec := sm.myProtoSpec + if targetSpec.ShardID == shard.BeaconChainShardID { // for beacon chain, only connect to beacon nodes + targetSpec.BeaconNode = true + } + return string(targetSpec.ToProtoID()) +} + func (sm *streamManager) setupStreamWithPeer(ctx context.Context, pid libp2p_peer.ID) error { timer := prometheus.NewTimer(setupStreamDuration.With(prometheus.Labels{"topic": string(sm.myProtoID)})) defer timer.ObserveDuration() @@ -347,7 +362,7 @@ func (sm *streamManager) setupStreamWithPeer(ctx context.Context, pid libp2p_pee nCtx, cancel := context.WithTimeout(ctx, connectTimeout) defer cancel() - st, err := sm.host.NewStream(nCtx, pid, protocol.ID(sm.myProtoID)) + st, err := sm.host.NewStream(nCtx, pid, protocol.ID(sm.targetProtoID())) if err != nil { return err } @@ -392,6 +407,10 @@ func (ss *streamSet) get(id sttypes.StreamID) (sttypes.Stream, bool) { ss.lock.RLock() defer ss.lock.RUnlock() + if id == "" { + return nil, false + } + st, ok := ss.streams[id] return st, ok } diff --git a/p2p/stream/protocols/sync/client.go b/p2p/stream/protocols/sync/client.go index 2523ef4f7e..7ad4798851 100644 --- a/p2p/stream/protocols/sync/client.go +++ b/p2p/stream/protocols/sync/client.go @@ -43,6 +43,36 @@ func (p *Protocol) GetBlocksByNumber(ctx context.Context, bns []uint64, opts ... return } +func (p *Protocol) GetRawBlocksByNumber(ctx context.Context, bns []uint64, opts ...Option) (blockBytes [][]byte, sigBytes [][]byte, stid sttypes.StreamID, err error) { + timer := p.doMetricClientRequest("getBlocksByNumber") + defer p.doMetricPostClientRequest("getBlocksByNumber", err, timer) + + if len(bns) == 0 { + err = fmt.Errorf("zero block numbers requested") + return + } + if len(bns) > GetBlocksByNumAmountCap { + err = fmt.Errorf("number of blocks exceed cap of %v", GetBlocksByNumAmountCap) + return + } + req := newGetBlocksByNumberRequest(bns) + resp, stid, err := p.rm.DoRequest(ctx, req, opts...) + if err != nil { + // At this point, error can be context canceled, context timed out, or waiting queue + // is already full. + return + } + + // Parse and return blocks + sResp, ok := resp.(*syncResponse) + if !ok || sResp == nil { + err = errors.New("not sync response") + return + } + blockBytes, sigBytes, err = req.parseBlockBytesAndSigs(sResp) + return +} + // GetCurrentBlockNumber get the current block number from remote node func (p *Protocol) GetCurrentBlockNumber(ctx context.Context, opts ...Option) (bn uint64, stid sttypes.StreamID, err error) { timer := p.doMetricClientRequest("getBlockNumber") diff --git a/p2p/stream/protocols/sync/const.go b/p2p/stream/protocols/sync/const.go index f536d4a78a..1e1fc612f0 100644 --- a/p2p/stream/protocols/sync/const.go +++ b/p2p/stream/protocols/sync/const.go @@ -17,6 +17,9 @@ const ( // See comments for GetBlocksByNumAmountCap. GetBlocksByHashesAmountCap = 10 + // MaxStreamFailures is the maximum allowed failures before stream gets removed + MaxStreamFailures = 3 + // minAdvertiseInterval is the minimum advertise interval minAdvertiseInterval = 1 * time.Minute diff --git a/p2p/stream/protocols/sync/protocol.go b/p2p/stream/protocols/sync/protocol.go index facdf601db..c63e6c45a5 100644 --- a/p2p/stream/protocols/sync/protocol.go +++ b/p2p/stream/protocols/sync/protocol.go @@ -2,11 +2,13 @@ package sync import ( "context" + "fmt" "strconv" "time" "github.com/ethereum/go-ethereum/event" "github.com/harmony-one/harmony/consensus/engine" + "github.com/harmony-one/harmony/core" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" "github.com/harmony-one/harmony/internal/utils" @@ -15,6 +17,7 @@ import ( "github.com/harmony-one/harmony/p2p/stream/common/requestmanager" "github.com/harmony-one/harmony/p2p/stream/common/streammanager" sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/harmony-one/harmony/shard" "github.com/hashicorp/go-version" libp2p_host "github.com/libp2p/go-libp2p/core/host" libp2p_network "github.com/libp2p/go-libp2p/core/network" @@ -39,12 +42,13 @@ var ( type ( // Protocol is the protocol for sync streaming Protocol struct { - chain engine.ChainReader // provide SYNC data - schedule shardingconfig.Schedule // provide schedule information - rl ratelimiter.RateLimiter // limit the incoming request rate - sm streammanager.StreamManager // stream management - rm requestmanager.RequestManager // deliver the response from stream - disc discovery.Discovery + chain engine.ChainReader // provide SYNC data + beaconNode bool // is beacon node or shard chain node + schedule shardingconfig.Schedule // provide schedule information + rl ratelimiter.RateLimiter // limit the incoming request rate + sm streammanager.StreamManager // stream management + rm requestmanager.RequestManager // deliver the response from stream + disc discovery.Discovery config Config logger zerolog.Logger @@ -74,13 +78,18 @@ type ( func NewProtocol(config Config) *Protocol { ctx, cancel := context.WithCancel(context.Background()) + isBeaconNode := config.Chain.ShardID() == shard.BeaconChainShardID + if _, ok := config.Chain.(*core.EpochChain); ok { + isBeaconNode = false + } sp := &Protocol{ - chain: config.Chain, - disc: config.Discovery, - config: config, - ctx: ctx, - cancel: cancel, - closeC: make(chan struct{}), + chain: config.Chain, + beaconNode: isBeaconNode, + disc: config.Discovery, + config: config, + ctx: ctx, + cancel: cancel, + closeC: make(chan struct{}), } smConfig := streammanager.Config{ SoftLoCap: config.SmSoftLowCap, @@ -162,6 +171,7 @@ func (p *Protocol) HandleStream(raw libp2p_network.Stream) { Msg("failed to add new stream") return } + fmt.Println("Node connected to", raw.Conn().RemotePeer().String(), "(", st.ProtoID(), ")") st.run() } @@ -219,18 +229,40 @@ func (p *Protocol) protoIDByVersion(v *version.Version) sttypes.ProtoID { NetworkType: p.config.Network, ShardID: p.config.ShardID, Version: v, + BeaconNode: p.beaconNode, } return spec.ToProtoID() } // RemoveStream removes the stream of the given stream ID +// TODO: add reason to parameters func (p *Protocol) RemoveStream(stID sttypes.StreamID) { - if stID == "" { - return - } st, exist := p.sm.GetStreamByID(stID) if exist && st != nil { + //TODO: log this incident with reason st.Close() + // stream manager removes this stream from the list and triggers discovery if number of streams are not enough + p.sm.RemoveStream(stID) //TODO: double check to see if this part is needed + } +} + +func (p *Protocol) StreamFailed(stID sttypes.StreamID, reason string) { + st, exist := p.sm.GetStreamByID(stID) + if exist && st != nil { + st.AddFailedTimes() + p.logger.Info(). + Str("stream ID", string(st.ID())). + Int("num failures", st.FailedTimes()). + Str("reason", reason). + Msg("stream failed") + if st.FailedTimes() >= MaxStreamFailures { + st.Close() + // stream manager removes this stream from the list and triggers discovery if number of streams are not enough + p.sm.RemoveStream(stID) //TODO: double check to see if this part is needed + p.logger.Warn(). + Str("stream ID", string(st.ID())). + Msg("stream removed") + } } } diff --git a/p2p/stream/types/stream.go b/p2p/stream/types/stream.go index 3abdf4f524..49860a2895 100644 --- a/p2p/stream/types/stream.go +++ b/p2p/stream/types/stream.go @@ -21,6 +21,9 @@ type Stream interface { ReadBytes() ([]byte, error) Close() error CloseOnExit() error + FailedTimes() int + AddFailedTimes() + ResetFailedRimes() } // BaseStream is the wrapper around @@ -34,14 +37,17 @@ type BaseStream struct { spec ProtoSpec specErr error specOnce sync.Once + + failedTimes int } // NewBaseStream creates BaseStream as the wrapper of libp2p Stream func NewBaseStream(st libp2p_network.Stream) *BaseStream { reader := bufio.NewReader(st) return &BaseStream{ - raw: st, - reader: reader, + raw: st, + reader: reader, + failedTimes: 0, } } @@ -72,6 +78,18 @@ func (st *BaseStream) Close() error { return st.raw.Reset() } +func (st *BaseStream) FailedTimes() int { + return st.failedTimes +} + +func (st *BaseStream) AddFailedTimes() { + st.failedTimes++ +} + +func (st *BaseStream) ResetFailedRimes() { + st.failedTimes = 0 +} + const ( maxMsgBytes = 20 * 1024 * 1024 // 20MB sizeBytes = 4 // uint32 diff --git a/p2p/stream/types/utils.go b/p2p/stream/types/utils.go index d096115fb7..bd8c6144af 100644 --- a/p2p/stream/types/utils.go +++ b/p2p/stream/types/utils.go @@ -11,7 +11,7 @@ import ( nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/hashicorp/go-version" - libp2p_proto "github.com/libp2p/go-libp2p/core/protocol" + libp2p_proto "github.com/libp2p/go-libp2p-core/protocol" "github.com/pkg/errors" ) @@ -20,10 +20,10 @@ const ( ProtoIDCommonPrefix = "harmony" // ProtoIDFormat is the format of stream protocol ID - ProtoIDFormat = "%s/%s/%s/%d/%s" + ProtoIDFormat = "%s/%s/%s/%d/%s/%d" // protoIDNumElem is the number of elements of the ProtoID. See comments in ProtoID - protoIDNumElem = 5 + protoIDNumElem = 6 ) // ProtoID is the protocol id for streaming, an alias of libp2p stream protocol ID。 @@ -32,6 +32,7 @@ const ( // 2. NetworkType - mainnet, testnet, stn, e.t.c. // 3. ShardID - shard ID of the current protocol. // 4. Version - Stream protocol version for backward compatibility. +// 5. BeaconNode - whether stream is from a beacon chain node or shard chain node type ProtoID libp2p_proto.ID // ProtoSpec is the un-serialized stream proto id specification @@ -43,12 +44,13 @@ type ProtoSpec struct { NetworkType nodeconfig.NetworkType ShardID nodeconfig.ShardID Version *version.Version + BeaconNode bool } // ToProtoID convert a ProtoSpec to ProtoID. func (spec ProtoSpec) ToProtoID() ProtoID { s := fmt.Sprintf(ProtoIDFormat, ProtoIDCommonPrefix, spec.Service, - spec.NetworkType, spec.ShardID, spec.Version.String()) + spec.NetworkType, spec.ShardID, spec.Version.String(), bool2int(spec.BeaconNode)) return ProtoID(s) } @@ -59,11 +61,12 @@ func ProtoIDToProtoSpec(id ProtoID) (ProtoSpec, error) { return ProtoSpec{}, errors.New("unexpected protocol size") } var ( - prefix = comps[0] - service = comps[1] - networkType = comps[2] - shardIDStr = comps[3] - versionStr = comps[4] + prefix = comps[0] + service = comps[1] + networkType = comps[2] + shardIDStr = comps[3] + versionStr = comps[4] + beaconnodeStr = comps[5] ) shardID, err := strconv.Atoi(shardIDStr) if err != nil { @@ -76,11 +79,16 @@ func ProtoIDToProtoSpec(id ProtoID) (ProtoSpec, error) { if err != nil { return ProtoSpec{}, errors.Wrap(err, "unexpected version string") } + isBeaconNode, err := strconv.Atoi(beaconnodeStr) + if err != nil { + return ProtoSpec{}, errors.Wrap(err, "invalid beacon node flag") + } return ProtoSpec{ Service: service, NetworkType: nodeconfig.NetworkType(networkType), ShardID: nodeconfig.ShardID(uint32(shardID)), Version: version, + BeaconNode: int2bool(isBeaconNode), }, nil } @@ -90,3 +98,14 @@ func GenReqID() uint64 { rand.Read(rnd[:]) return binary.BigEndian.Uint64(rnd[:]) } + +func bool2int(b bool) int { + if b { + return 1 + } + return 0 +} + +func int2bool(i int) bool { + return i > 0 +} From 4617ef756357304266f2636827664a16d5db1072 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <“Gheis.Mohammadi@gmail.com”> Date: Tue, 27 Dec 2022 18:01:25 +0800 Subject: [PATCH 171/420] fix protocol tests --- api/service/stagedstreamsync/downloader.go | 3 ++- api/service/stagedstreamsync/staged_stream_sync.go | 6 +++--- p2p/stream/common/streammanager/interface_test.go | 2 +- p2p/stream/common/streammanager/streammanager.go | 2 +- .../common/streammanager/streammanager_test.go | 2 +- p2p/stream/protocols/sync/protocol_test.go | 12 +++++++----- p2p/stream/types/utils.go | 6 +++++- 7 files changed, 20 insertions(+), 13 deletions(-) diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go index 9fdd2f78f4..383447f982 100644 --- a/api/service/stagedstreamsync/downloader.go +++ b/api/service/stagedstreamsync/downloader.go @@ -32,7 +32,7 @@ type ( downloadC chan struct{} closeC chan struct{} ctx context.Context - cancel func() + cancel context.CancelFunc config Config logger zerolog.Logger @@ -70,6 +70,7 @@ func NewDownloader(host p2p.Host, bc core.BlockChain, config Config) *Downloader //TODO: use mem db should be in config file stagedSyncInstance, err := CreateStagedSync(ctx, bc, false, sp, config, logger, config.LogProgress) if err != nil { + cancel() return nil } diff --git a/api/service/stagedstreamsync/staged_stream_sync.go b/api/service/stagedstreamsync/staged_stream_sync.go index 1abd728498..11ad9e8aee 100644 --- a/api/service/stagedstreamsync/staged_stream_sync.go +++ b/api/service/stagedstreamsync/staged_stream_sync.go @@ -65,8 +65,8 @@ type StagedStreamSync struct { inserted int config Config logger zerolog.Logger - status status //TODO: merge this with currentSyncCycle - initSync bool // if sets to true, node start long range syncing + status *status //TODO: merge this with currentSyncCycle + initSync bool // if sets to true, node start long range syncing UseMemDB bool revertPoint *uint64 // used to run stages @@ -294,7 +294,7 @@ func New(ctx context.Context, db: db, protocol: protocol, gbm: nil, - status: status, + status: &status, inserted: 0, config: config, logger: logger, diff --git a/p2p/stream/common/streammanager/interface_test.go b/p2p/stream/common/streammanager/interface_test.go index e308c3abab..033273fd67 100644 --- a/p2p/stream/common/streammanager/interface_test.go +++ b/p2p/stream/common/streammanager/interface_test.go @@ -17,7 +17,7 @@ var _ StreamManager = &streamManager{} var ( myPeerID = makePeerID(0) - testProtoID = sttypes.ProtoID("harmony/sync/unitest/0/1.0.0") + testProtoID = sttypes.ProtoID("harmony/sync/unitest/0/1.0.0/1") ) const ( diff --git a/p2p/stream/common/streammanager/streammanager.go b/p2p/stream/common/streammanager/streammanager.go index 33270e1c7b..a341184981 100644 --- a/p2p/stream/common/streammanager/streammanager.go +++ b/p2p/stream/common/streammanager/streammanager.go @@ -410,7 +410,7 @@ func (ss *streamSet) get(id sttypes.StreamID) (sttypes.Stream, bool) { if id == "" { return nil, false } - + st, ok := ss.streams[id] return st, ok } diff --git a/p2p/stream/common/streammanager/streammanager_test.go b/p2p/stream/common/streammanager/streammanager_test.go index 5d82c8585f..2b49a5f16c 100644 --- a/p2p/stream/common/streammanager/streammanager_test.go +++ b/p2p/stream/common/streammanager/streammanager_test.go @@ -209,7 +209,7 @@ func TestStreamSet_numStreamsWithMinProtoID(t *testing.T) { pid1 = testProtoID numPid1 = 5 - pid2 = sttypes.ProtoID("harmony/sync/unitest/0/1.0.1") + pid2 = sttypes.ProtoID("harmony/sync/unitest/0/1.0.1/1") numPid2 = 10 ) diff --git a/p2p/stream/protocols/sync/protocol_test.go b/p2p/stream/protocols/sync/protocol_test.go index aff6691ec8..0e40f60175 100644 --- a/p2p/stream/protocols/sync/protocol_test.go +++ b/p2p/stream/protocols/sync/protocol_test.go @@ -15,16 +15,18 @@ func TestProtocol_Match(t *testing.T) { targetID string exp bool }{ - {"harmony/sync/unitest/0/1.0.1", true}, + {"harmony/sync/unitest/0/1.0.1/1", true}, + {"harmony/sync/unitest/0/1.0.1/0", true}, {"h123456", false}, - {"harmony/sync/unitest/0/0.9.9", false}, - {"harmony/epoch/unitest/0/1.0.1", false}, - {"harmony/sync/mainnet/0/1.0.1", false}, - {"harmony/sync/unitest/1/1.0.1", false}, + {"harmony/sync/unitest/0/0.9.9/1", false}, + {"harmony/epoch/unitest/0/1.0.1/1", false}, + {"harmony/sync/mainnet/0/1.0.1/1", false}, + {"harmony/sync/unitest/1/1.0.1/1", false}, } for i, test := range tests { p := &Protocol{ + beaconNode: true, config: Config{ Network: "unitest", ShardID: 0, diff --git a/p2p/stream/types/utils.go b/p2p/stream/types/utils.go index bd8c6144af..86eb882183 100644 --- a/p2p/stream/types/utils.go +++ b/p2p/stream/types/utils.go @@ -49,8 +49,12 @@ type ProtoSpec struct { // ToProtoID convert a ProtoSpec to ProtoID. func (spec ProtoSpec) ToProtoID() ProtoID { + var versionStr string + if spec.Version != nil { + versionStr = spec.Version.String() + } s := fmt.Sprintf(ProtoIDFormat, ProtoIDCommonPrefix, spec.Service, - spec.NetworkType, spec.ShardID, spec.Version.String(), bool2int(spec.BeaconNode)) + spec.NetworkType, spec.ShardID, versionStr, bool2int(spec.BeaconNode)) return ProtoID(s) } From 57108cf7bf34f4af866ed3864d7bd39e2d987b77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <“Gheis.Mohammadi@gmail.com”> Date: Tue, 27 Dec 2022 18:26:22 +0800 Subject: [PATCH 172/420] fix spell --- p2p/stream/common/requestmanager/interface_test.go | 2 +- p2p/stream/common/streammanager/interface_test.go | 2 +- p2p/stream/types/stream.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/p2p/stream/common/requestmanager/interface_test.go b/p2p/stream/common/requestmanager/interface_test.go index 76399e859e..c51303ccba 100644 --- a/p2p/stream/common/requestmanager/interface_test.go +++ b/p2p/stream/common/requestmanager/interface_test.go @@ -122,7 +122,7 @@ func (st *testStream) AddFailedTimes() { return } -func (st *testStream) ResetFailedRimes() { +func (st *testStream) ResetFailedTimes() { return } diff --git a/p2p/stream/common/streammanager/interface_test.go b/p2p/stream/common/streammanager/interface_test.go index 033273fd67..5a9bb44366 100644 --- a/p2p/stream/common/streammanager/interface_test.go +++ b/p2p/stream/common/streammanager/interface_test.go @@ -78,7 +78,7 @@ func (st *testStream) AddFailedTimes() { return } -func (st *testStream) ResetFailedRimes() { +func (st *testStream) ResetFailedTimes() { return } diff --git a/p2p/stream/types/stream.go b/p2p/stream/types/stream.go index 49860a2895..18b47f6158 100644 --- a/p2p/stream/types/stream.go +++ b/p2p/stream/types/stream.go @@ -23,7 +23,7 @@ type Stream interface { CloseOnExit() error FailedTimes() int AddFailedTimes() - ResetFailedRimes() + ResetFailedTimes() } // BaseStream is the wrapper around @@ -86,7 +86,7 @@ func (st *BaseStream) AddFailedTimes() { st.failedTimes++ } -func (st *BaseStream) ResetFailedRimes() { +func (st *BaseStream) ResetFailedTimes() { st.failedTimes = 0 } From ac6bfd19c5f032151275ffcefb550de7b2829c54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <“Gheis.Mohammadi@gmail.com”> Date: Tue, 27 Dec 2022 19:50:09 +0800 Subject: [PATCH 173/420] remove unused struct --- api/service/stagedstreamsync/staged_stream_sync.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/api/service/stagedstreamsync/staged_stream_sync.go b/api/service/stagedstreamsync/staged_stream_sync.go index 11ad9e8aee..a5fa011f43 100644 --- a/api/service/stagedstreamsync/staged_stream_sync.go +++ b/api/service/stagedstreamsync/staged_stream_sync.go @@ -9,7 +9,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/event" "github.com/harmony-one/harmony/core" - "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/utils" syncproto "github.com/harmony-one/harmony/p2p/stream/protocols/sync" sttypes "github.com/harmony-one/harmony/p2p/stream/types" @@ -87,13 +86,6 @@ type StagedStreamSync struct { evtDownloadStartedSubscribed bool } -// BlockWithSig the serialization structure for request DownloaderRequest_BLOCKWITHSIG -// The block is encoded as block + commit signature -type BlockWithSig struct { - Block *types.Block - CommitSigAndBitmap []byte -} - type Timing struct { isRevert bool isCleanUp bool @@ -185,7 +177,7 @@ func (s *StagedStreamSync) IsAfter(stage1, stage2 SyncStageID) bool { func (s *StagedStreamSync) RevertTo(revertPoint uint64, invalidBlockNumber uint64, invalidBlockHash common.Hash, invalidBlockStreamID sttypes.StreamID) { utils.Logger().Info(). - Interface("invalidBlockNumber", invalidBlockNumber). + Uint64("invalidBlockNumber", invalidBlockNumber). Interface("invalidBlockHash", invalidBlockHash). Interface("invalidBlockStreamID", invalidBlockStreamID). Uint64("revertPoint", revertPoint). From 7542ac574340224b538d87b3dadb90fb2dfe1931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <“Gheis.Mohammadi@gmail.com”> Date: Wed, 28 Dec 2022 13:34:35 +0800 Subject: [PATCH 174/420] fix rosetta test --- rosetta/infra/harmony-mainnet.conf | 3 ++- rosetta/infra/harmony-pstn.conf | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/rosetta/infra/harmony-mainnet.conf b/rosetta/infra/harmony-mainnet.conf index 534b80887f..ddb2f87eea 100644 --- a/rosetta/infra/harmony-mainnet.conf +++ b/rosetta/infra/harmony-mainnet.conf @@ -1,4 +1,4 @@ -Version = "2.5.9" +Version = "2.5.10" [BLSKeys] KMSConfigFile = "" @@ -68,6 +68,7 @@ Version = "2.5.9" MaxConnsPerIP = 10 MaxPeers = 0 Port = 9000 + WaitForEachPeerToConnect = false [Pprof] Enabled = false diff --git a/rosetta/infra/harmony-pstn.conf b/rosetta/infra/harmony-pstn.conf index ed4c116c6b..1c029f8281 100644 --- a/rosetta/infra/harmony-pstn.conf +++ b/rosetta/infra/harmony-pstn.conf @@ -1,4 +1,4 @@ -Version = "2.5.9" +Version = "2.5.10" [BLSKeys] KMSConfigFile = "" @@ -68,6 +68,7 @@ Version = "2.5.9" MaxConnsPerIP = 10 MaxPeers = 0 Port = 9000 + WaitForEachPeerToConnect = false [Pprof] Enabled = false From 9ea049b519b2ad61a1aad5b304063f20ccc06ca9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <“Gheis.Mohammadi@gmail.com”> Date: Wed, 28 Dec 2022 18:41:09 +0800 Subject: [PATCH 175/420] add comments and refactor verify sig --- api/service/stagedstreamsync/const.go | 4 +- api/service/stagedstreamsync/downloader.go | 56 +---------------- api/service/stagedstreamsync/helpers.go | 14 +++++ api/service/stagedstreamsync/sig_verify.go | 60 +++++++++++++++++++ .../stagedstreamsync/staged_stream_sync.go | 32 +++++----- 5 files changed, 95 insertions(+), 71 deletions(-) create mode 100644 api/service/stagedstreamsync/sig_verify.go diff --git a/api/service/stagedstreamsync/const.go b/api/service/stagedstreamsync/const.go index 721b44d9f5..ed68d80b13 100644 --- a/api/service/stagedstreamsync/const.go +++ b/api/service/stagedstreamsync/const.go @@ -21,11 +21,11 @@ const ( SoftQueueCap int = 100 // DefaultConcurrency is the default settings for concurrency - DefaultConcurrency = 4 + DefaultConcurrency int = 4 // ShortRangeTimeout is the timeout for each short range sync, which allow short range sync // to restart automatically when stuck in `getBlockHashes` - ShortRangeTimeout = 1 * time.Minute + ShortRangeTimeout time.Duration = 1 * time.Minute ) type ( diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go index 383447f982..9883c8e8f7 100644 --- a/api/service/stagedstreamsync/downloader.go +++ b/api/service/stagedstreamsync/downloader.go @@ -6,13 +6,9 @@ import ( "time" "github.com/ethereum/go-ethereum/event" - "github.com/pkg/errors" "github.com/rs/zerolog" "github.com/harmony-one/harmony/core" - "github.com/harmony-one/harmony/core/types" - "github.com/harmony-one/harmony/crypto/bls" - "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" @@ -128,7 +124,7 @@ func (d *Downloader) NumPeers() int { return d.syncProtocol.NumStreams() } -// IsSyncing returns the current sync status +// SyncStatus returns the current sync status func (d *Downloader) SyncStatus() (bool, uint64, uint64) { syncing, target := d.stagedSyncInstance.status.get() if !syncing { @@ -252,53 +248,3 @@ func (d *Downloader) loop() { } } } - -var emptySigVerifyErr *sigVerifyErr - -type sigVerifyErr struct { - err error -} - -func (e *sigVerifyErr) Error() string { - return fmt.Sprintf("[VerifyHeaderSignature] %v", e.err.Error()) -} - -func verifyAndInsertBlocks(bc blockChain, blocks types.Blocks) (int, error) { - for i, block := range blocks { - if err := verifyAndInsertBlock(bc, block, blocks[i+1:]...); err != nil { - return i, err - } - } - return len(blocks), nil -} - -func verifyAndInsertBlock(bc blockChain, block *types.Block, nextBlocks ...*types.Block) error { - var ( - sigBytes bls.SerializedSignature - bitmap []byte - err error - ) - if len(nextBlocks) > 0 { - // get commit sig from the next block - next := nextBlocks[0] - sigBytes = next.Header().LastCommitSignature() - bitmap = next.Header().LastCommitBitmap() - } else { - // get commit sig from current block - sigBytes, bitmap, err = chain.ParseCommitSigAndBitmap(block.GetCurrentCommitSig()) - if err != nil { - return errors.Wrap(err, "parse commitSigAndBitmap") - } - } - - if err := bc.Engine().VerifyHeaderSignature(bc, block.Header(), sigBytes, bitmap); err != nil { - return &sigVerifyErr{err} - } - if err := bc.Engine().VerifyHeader(bc, block.Header(), true); err != nil { - return errors.Wrap(err, "[VerifyHeader]") - } - if _, err := bc.InsertChain(types.Blocks{block}, false); err != nil { - return errors.Wrap(err, "[InsertChain]") - } - return nil -} diff --git a/api/service/stagedstreamsync/helpers.go b/api/service/stagedstreamsync/helpers.go index cd6fd8f6f0..9e986a538b 100644 --- a/api/service/stagedstreamsync/helpers.go +++ b/api/service/stagedstreamsync/helpers.go @@ -112,3 +112,17 @@ func countHashMaxVote(m map[sttypes.StreamID]common.Hash, whitelist map[sttypes. } return res, nextWl } + +func ByteCount(b uint64) string { + const unit = 1024 + if b < unit { + return fmt.Sprintf("%dB", b) + } + div, exp := uint64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f%cB", + float64(b)/float64(div), "KMGTPE"[exp]) +} diff --git a/api/service/stagedstreamsync/sig_verify.go b/api/service/stagedstreamsync/sig_verify.go new file mode 100644 index 0000000000..649c6eaec1 --- /dev/null +++ b/api/service/stagedstreamsync/sig_verify.go @@ -0,0 +1,60 @@ +package stagedstreamsync + +import ( + "fmt" + + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/crypto/bls" + "github.com/harmony-one/harmony/internal/chain" + "github.com/pkg/errors" +) + +var emptySigVerifyErr *sigVerifyErr + +type sigVerifyErr struct { + err error +} + +func (e *sigVerifyErr) Error() string { + return fmt.Sprintf("[VerifyHeaderSignature] %v", e.err.Error()) +} + +func verifyAndInsertBlocks(bc blockChain, blocks types.Blocks) (int, error) { + for i, block := range blocks { + if err := verifyAndInsertBlock(bc, block, blocks[i+1:]...); err != nil { + return i, err + } + } + return len(blocks), nil +} + +func verifyAndInsertBlock(bc blockChain, block *types.Block, nextBlocks ...*types.Block) error { + var ( + sigBytes bls.SerializedSignature + bitmap []byte + err error + ) + if len(nextBlocks) > 0 { + // get commit sig from the next block + next := nextBlocks[0] + sigBytes = next.Header().LastCommitSignature() + bitmap = next.Header().LastCommitBitmap() + } else { + // get commit sig from current block + sigBytes, bitmap, err = chain.ParseCommitSigAndBitmap(block.GetCurrentCommitSig()) + if err != nil { + return errors.Wrap(err, "parse commitSigAndBitmap") + } + } + + if err := bc.Engine().VerifyHeaderSignature(bc, block.Header(), sigBytes, bitmap); err != nil { + return &sigVerifyErr{err} + } + if err := bc.Engine().VerifyHeader(bc, block.Header(), true); err != nil { + return errors.Wrap(err, "[VerifyHeader]") + } + if _, err := bc.InsertChain(types.Blocks{block}, false); err != nil { + return errors.Wrap(err, "[InsertChain]") + } + return nil +} diff --git a/api/service/stagedstreamsync/staged_stream_sync.go b/api/service/stagedstreamsync/staged_stream_sync.go index a5fa011f43..9603f5b06b 100644 --- a/api/service/stagedstreamsync/staged_stream_sync.go +++ b/api/service/stagedstreamsync/staged_stream_sync.go @@ -175,6 +175,7 @@ func (s *StagedStreamSync) IsAfter(stage1, stage2 SyncStageID) bool { return idx1 > idx2 } +// RevertTo sets the revert point func (s *StagedStreamSync) RevertTo(revertPoint uint64, invalidBlockNumber uint64, invalidBlockHash common.Hash, invalidBlockStreamID sttypes.StreamID) { utils.Logger().Info(). Uint64("invalidBlockNumber", invalidBlockNumber). @@ -195,10 +196,12 @@ func (s *StagedStreamSync) Done() { s.revertPoint = nil } +// IsDone returns true if last stage have been done func (s *StagedStreamSync) IsDone() bool { return s.currentStage >= uint(len(s.stages)) && s.revertPoint == nil } +// SetCurrentStage sets the current stage to a given stage id func (s *StagedStreamSync) SetCurrentStage(id SyncStageID) error { for i, stage := range s.stages { if stage.ID == id { @@ -210,6 +213,7 @@ func (s *StagedStreamSync) SetCurrentStage(id SyncStageID) error { return ErrStageNotFound } +// StageState retrieves the latest stage state from db func (s *StagedStreamSync) StageState(stage SyncStageID, tx kv.Tx, db kv.RwDB) (*StageState, error) { var blockNum uint64 var err error @@ -226,6 +230,7 @@ func (s *StagedStreamSync) StageState(stage SyncStageID, tx kv.Tx, db kv.RwDB) ( return &StageState{s, stage, blockNum}, nil } +// cleanUp cleans up the stage by calling pruneStage func (s *StagedStreamSync) cleanUp(fromStage int, db kv.RwDB, tx kv.RwTx, firstCycle bool) error { found := false for i := 0; i < len(s.pruningOrder); i++ { @@ -242,6 +247,7 @@ func (s *StagedStreamSync) cleanUp(fromStage int, db kv.RwDB, tx kv.RwTx, firstC return nil } +// New creates a new StagedStreamSync instance func New(ctx context.Context, bc core.BlockChain, db kv.RwDB, @@ -299,6 +305,7 @@ func New(ctx context.Context, } } +// doGetCurrentNumberRequest returns estimated current block number and corresponding stream func (s *StagedStreamSync) doGetCurrentNumberRequest() (uint64, sttypes.StreamID, error) { ctx, cancel := context.WithTimeout(s.ctx, 10*time.Second) defer cancel() @@ -310,11 +317,13 @@ func (s *StagedStreamSync) doGetCurrentNumberRequest() (uint64, sttypes.StreamID return bn, stid, nil } +// promLabels returns a prometheus labels for current shard id func (s *StagedStreamSync) promLabels() prometheus.Labels { sid := s.bc.ShardID() return prometheus.Labels{"ShardID": fmt.Sprintf("%d", sid)} } +// checkHaveEnoughStreams checks whether node is connected to certain number of streams func (s *StagedStreamSync) checkHaveEnoughStreams() error { numStreams := s.protocol.NumStreams() if numStreams < s.config.MinStreams { @@ -324,6 +333,7 @@ func (s *StagedStreamSync) checkHaveEnoughStreams() error { return nil } +// SetNewContext sets a new context for all stages func (s *StagedStreamSync) SetNewContext(ctx context.Context) error { for _, s := range s.stages { s.Handler.SetStageContext(ctx) @@ -331,6 +341,7 @@ func (s *StagedStreamSync) SetNewContext(ctx context.Context) error { return nil } +// Run runs a full cycle of stages func (s *StagedStreamSync) Run(db kv.RwDB, tx kv.RwTx, firstCycle bool) error { s.prevRevertPoint = nil s.timings = s.timings[:0] @@ -395,6 +406,7 @@ func (s *StagedStreamSync) Run(db kv.RwDB, tx kv.RwTx, firstCycle bool) error { return nil } +// CreateView creates a view for a given db func CreateView(ctx context.Context, db kv.RwDB, tx kv.Tx, f func(tx kv.Tx) error) error { if tx != nil { return f(tx) @@ -404,20 +416,7 @@ func CreateView(ctx context.Context, db kv.RwDB, tx kv.Tx, f func(tx kv.Tx) erro }) } -func ByteCount(b uint64) string { - const unit = 1024 - if b < unit { - return fmt.Sprintf("%dB", b) - } - div, exp := uint64(unit), 0 - for n := b / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f%cB", - float64(b)/float64(div), "KMGTPE"[exp]) -} - +// printLogs prints all timing logs func printLogs(tx kv.RwTx, timings []Timing) error { var logCtx []interface{} count := 0 @@ -463,6 +462,7 @@ func printLogs(tx kv.RwTx, timings []Timing) error { return nil } +// runStage executes stage func (s *StagedStreamSync) runStage(stage *Stage, db kv.RwDB, tx kv.RwTx, firstCycle bool, invalidBlockRevert bool) (err error) { start := time.Now() stageState, err := s.StageState(stage.ID, tx, db) @@ -489,6 +489,7 @@ func (s *StagedStreamSync) runStage(stage *Stage, db kv.RwDB, tx kv.RwTx, firstC return nil } +// revertStage reverts stage func (s *StagedStreamSync) revertStage(firstCycle bool, stage *Stage, db kv.RwDB, tx kv.RwTx) error { start := time.Now() stageState, err := s.StageState(stage.ID, tx, db) @@ -521,6 +522,7 @@ func (s *StagedStreamSync) revertStage(firstCycle bool, stage *Stage, db kv.RwDB return nil } +// pruneStage cleans up the stage and logs the timing func (s *StagedStreamSync) pruneStage(firstCycle bool, stage *Stage, db kv.RwDB, tx kv.RwTx) error { start := time.Now() @@ -566,6 +568,7 @@ func (s *StagedStreamSync) DisableAllStages() []SyncStageID { return backupEnabledIds } +// DisableStages disables stages by a set of given stage IDs func (s *StagedStreamSync) DisableStages(ids ...SyncStageID) { for i := range s.stages { for _, id := range ids { @@ -577,6 +580,7 @@ func (s *StagedStreamSync) DisableStages(ids ...SyncStageID) { } } +// EnableStages enables stages by a set of given stage IDs func (s *StagedStreamSync) EnableStages(ids ...SyncStageID) { for i := range s.stages { for _, id := range ids { From 1bd71f6e15d9a2966c586f7b9c32b4a263f3cd89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Wed, 28 Dec 2022 19:02:37 +0800 Subject: [PATCH 176/420] add comments, remove extra function --- api/service/stagedstreamsync/stages.go | 11 +++-------- api/service/stagedstreamsync/syncing.go | 1 + 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/api/service/stagedstreamsync/stages.go b/api/service/stagedstreamsync/stages.go index dc607416b8..55681d68f5 100644 --- a/api/service/stagedstreamsync/stages.go +++ b/api/service/stagedstreamsync/stages.go @@ -16,6 +16,7 @@ const ( Finish SyncStageID = "Finish" // Nominal stage after all other stages ) +// GetStageName returns the stage name in string func GetStageName(stage string, isBeacon bool, prune bool) string { name := stage if isBeacon { @@ -27,18 +28,11 @@ func GetStageName(stage string, isBeacon bool, prune bool) string { return name } +// GetStageID returns the stage name in bytes func GetStageID(stage SyncStageID, isBeacon bool, prune bool) []byte { return []byte(GetStageName(string(stage), isBeacon, prune)) } -func GetBucketName(bucketName string, isBeacon bool) string { - name := bucketName - if isBeacon { - name = "Beacon" + name - } - return name -} - // GetStageProgress retrieves saved progress of a given sync stage from the database func GetStageProgress(db kv.Getter, stage SyncStageID, isBeacon bool) (uint64, error) { stgID := GetStageID(stage, isBeacon, false) @@ -65,6 +59,7 @@ func GetStageCleanUpProgress(db kv.Getter, stage SyncStageID, isBeacon bool) (ui return unmarshalData(v) } +// SaveStageCleanUpProgress stores the progress of the clean up for a given sync stage to the database func SaveStageCleanUpProgress(db kv.Putter, stage SyncStageID, isBeacon bool, progress uint64) error { stgID := GetStageID(stage, isBeacon, true) return db.Put(kv.SyncStageProgress, stgID, marshalData(progress)) diff --git a/api/service/stagedstreamsync/syncing.go b/api/service/stagedstreamsync/syncing.go index 676eb3551e..046ba6843b 100644 --- a/api/service/stagedstreamsync/syncing.go +++ b/api/service/stagedstreamsync/syncing.go @@ -134,6 +134,7 @@ func initDB(ctx context.Context, mainDB kv.RwDB, dbs []kv.RwDB, concurrency int) return nil } +// GetBlockDbPath returns the path of the cache database which stores blocks func GetBlockDbPath(beacon bool, loopID int) string { if beacon { if loopID >= 0 { From e8ef1ea2198dc87e8d3cd18dc5d4c7c04bee156a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Wed, 28 Dec 2022 19:19:27 +0800 Subject: [PATCH 177/420] add comment --- api/service/stagedstreamsync/stage.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/service/stagedstreamsync/stage.go b/api/service/stagedstreamsync/stage.go index 8f2040a789..832cf4e3c7 100644 --- a/api/service/stagedstreamsync/stage.go +++ b/api/service/stagedstreamsync/stage.go @@ -96,6 +96,7 @@ func (u *RevertState) Done(db kv.Putter) error { return SaveStageProgress(db, u.ID, u.state.isBeacon, u.RevertPoint) } +// CleanUpState contains states of cleanup process for a specific stage type CleanUpState struct { ID SyncStageID ForwardProgress uint64 // progress of stage forward move From b33a8640a01d7a7030a5cbe379954b081dd0eecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Thu, 29 Dec 2022 11:36:58 +0800 Subject: [PATCH 178/420] refactor errors, rename metrics --- api/service/stagedstreamsync/beacon_helper.go | 9 ++-- .../stagedstreamsync/block_by_hash_manager.go | 2 +- api/service/stagedstreamsync/downloader.go | 4 +- api/service/stagedstreamsync/errors.go | 53 ++++++------------- api/service/stagedstreamsync/helpers.go | 5 +- api/service/stagedstreamsync/metric.go | 14 ++--- .../stagedstreamsync/short_range_helper.go | 11 ++-- api/service/stagedstreamsync/stage.go | 4 -- api/service/stagedstreamsync/stage_bodies.go | 2 +- api/service/stagedstreamsync/syncing.go | 2 +- api/service/stagedstreamsync/types.go | 2 +- hmy/downloader/metric.go | 14 ++--- 12 files changed, 51 insertions(+), 71 deletions(-) diff --git a/api/service/stagedstreamsync/beacon_helper.go b/api/service/stagedstreamsync/beacon_helper.go index 4a77020160..2f684e0e27 100644 --- a/api/service/stagedstreamsync/beacon_helper.go +++ b/api/service/stagedstreamsync/beacon_helper.go @@ -42,7 +42,7 @@ func newBeaconHelper(bc blockChain, blockC <-chan *types.Block, insertHook func( insertC: make(chan insertTask, 1), closeC: make(chan struct{}), logger: utils.Logger().With(). - Str("module", "downloader"). + Str("module", "staged stream sync"). Str("sub-module", "beacon helper"). Logger(), } @@ -78,7 +78,8 @@ func (bh *beaconHelper) loop() { inserted, bn, err := bh.insertLastMileBlocks() numBlocksInsertedBeaconHelperCounter.Add(float64(inserted)) if err != nil { - bh.logger.Error().Err(err).Msg(WrapStagedSyncMsg("insert last mile blocks error")) + bh.logger.Error().Err(err). + Msg(WrapStagedSyncMsg("insert last mile blocks error")) continue } bh.logger.Info().Int("inserted", inserted). @@ -127,7 +128,9 @@ func (bh *beaconHelper) insertLastMileBlocks() (inserted int, bn uint64, err err bn-- return } - bh.logger.Info().Uint64("number", b.NumberU64()).Msg(WrapStagedSyncMsg("Inserted block from beacon pub-sub")) + bh.logger.Info(). + Uint64("number", b.NumberU64()). + Msg(WrapStagedSyncMsg("Inserted block from beacon pub-sub")) if bh.insertHook != nil { bh.insertHook() diff --git a/api/service/stagedstreamsync/block_by_hash_manager.go b/api/service/stagedstreamsync/block_by_hash_manager.go index 86abc22ed7..4179bbaed1 100644 --- a/api/service/stagedstreamsync/block_by_hash_manager.go +++ b/api/service/stagedstreamsync/block_by_hash_manager.go @@ -34,7 +34,7 @@ func (m *getBlocksByHashManager) getNextHashes() ([]common.Hash, []sttypes.Strea num := m.numBlocksPerRequest() hashes := make([]common.Hash, 0, num) if len(m.whitelist) == 0 { - return nil, nil, errors.New("empty white list") + return nil, nil, ErrEmptyWhitelist } for _, hash := range m.hashes { diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go index 9883c8e8f7..bb3b8c82eb 100644 --- a/api/service/stagedstreamsync/downloader.go +++ b/api/service/stagedstreamsync/downloader.go @@ -59,7 +59,9 @@ func NewDownloader(host p2p.Host, bc core.BlockChain, config Config) *Downloader bh = newBeaconHelper(bc, config.BHConfig.BlockC, config.BHConfig.InsertHook) } - logger := utils.Logger().With().Str("module", "StagedStreamSync").Uint32("ShardID", bc.ShardID()).Logger() + logger := utils.Logger().With(). + Str("module", "staged stream sync"). + Uint32("ShardID", bc.ShardID()).Logger() ctx, cancel := context.WithCancel(context.Background()) diff --git a/api/service/stagedstreamsync/errors.go b/api/service/stagedstreamsync/errors.go index e4828003d2..d18020dd06 100644 --- a/api/service/stagedstreamsync/errors.go +++ b/api/service/stagedstreamsync/errors.go @@ -6,43 +6,22 @@ import ( // Errors ... var ( - ErrRegistrationFail = WrapStagedSyncError("registration failed") - ErrGetBlock = WrapStagedSyncError("get block failed") - ErrGetBlockHash = WrapStagedSyncError("get block hash failed") - ErrGetConsensusHashes = WrapStagedSyncError("get consensus hashes failed") - ErrGenStateSyncTaskQueue = WrapStagedSyncError("generate state sync task queue failed") - ErrDownloadBlocks = WrapStagedSyncError("get download blocks failed") - ErrUpdateBlockAndStatus = WrapStagedSyncError("update block and status failed") - ErrGenerateNewState = WrapStagedSyncError("get generate new state failed") - ErrFetchBlockHashProgressFail = WrapStagedSyncError("fetch cache progress for block hashes stage failed") - ErrFetchCachedBlockHashFail = WrapStagedSyncError("fetch cached block hashes failed") - ErrNotEnoughBlockHashes = WrapStagedSyncError("peers haven't sent all requested block hashes") - ErrRetrieveCachedProgressFail = WrapStagedSyncError("retrieving cache progress for block hashes stage failed") - ErrRetrieveCachedHashProgressFail = WrapStagedSyncError("retrieving cache progress for block hashes stage failed") - ErrSaveBlockHashesProgressFail = WrapStagedSyncError("saving progress for block hashes stage failed") - ErrSaveCachedBlockHashesProgressFail = WrapStagedSyncError("saving cache progress for block hashes stage failed") - ErrSavingCacheLastBlockHashFail = WrapStagedSyncError("saving cache last block hash for block hashes stage failed") - ErrCachingBlockHashFail = WrapStagedSyncError("caching downloaded block hashes failed") - ErrCommitTransactionFail = WrapStagedSyncError("failed to write db commit") - ErrUnexpectedNumberOfBlocks = WrapStagedSyncError("unexpected number of block delivered") - ErrSavingBodiesProgressFail = WrapStagedSyncError("saving progress for block bodies stage failed") - ErrAddTasksToQueueFail = WrapStagedSyncError("cannot add task to queue") - ErrSavingCachedBodiesProgressFail = WrapStagedSyncError("saving cache progress for blocks stage failed") - ErrRetrievingCachedBodiesProgressFail = WrapStagedSyncError("retrieving cache progress for blocks stage failed") - ErrNoConnectedPeers = WrapStagedSyncError("haven't connected to any peer yet!") - ErrNotEnoughConnectedPeers = WrapStagedSyncError("not enough connected peers") - ErrSaveStateProgressFail = WrapStagedSyncError("saving progress for block States stage failed") - ErrPruningCursorCreationFail = WrapStagedSyncError("failed to create cursor for pruning") - ErrInvalidBlockNumber = WrapStagedSyncError("invalid block number") - ErrInvalidBlockBytes = WrapStagedSyncError("invalid block bytes to insert into chain") - ErrAddTaskFailed = WrapStagedSyncError("cannot add task to queue") - ErrNodeNotEnoughBlockHashes = WrapStagedSyncError("some of the nodes didn't provide all block hashes") - ErrCachingBlocksFail = WrapStagedSyncError("caching downloaded block bodies failed") - ErrSaveBlocksFail = WrapStagedSyncError("save downloaded block bodies failed") - ErrStageNotFound = WrapStagedSyncError("stage not found") - ErrSomeNodesNotReady = WrapStagedSyncError("some nodes are not ready") - ErrSomeNodesBlockHashFail = WrapStagedSyncError("some nodes failed to download block hashes") - ErrMaxPeerHeightFail = WrapStagedSyncError("get max peer height failed") + ErrSavingBodiesProgressFail = WrapStagedSyncError("saving progress for block bodies stage failed") + ErrSaveStateProgressFail = WrapStagedSyncError("saving progress for block States stage failed") + ErrInvalidBlockNumber = WrapStagedSyncError("invalid block number") + ErrInvalidBlockBytes = WrapStagedSyncError("invalid block bytes to insert into chain") + ErrStageNotFound = WrapStagedSyncError("stage not found") + ErrUnexpectedNumberOfBlockHashes = WrapStagedSyncError("unexpected number of getBlocksByHashes result") + ErrUnexpectedBlockHashes = WrapStagedSyncError("unexpected get block hashes result delivered") + ErrNilBlock = WrapStagedSyncError("nil block found") + ErrNotEnoughStreams = WrapStagedSyncError("not enough streams") + ErrParseCommitSigAndBitmapFail = WrapStagedSyncError("parse commitSigAndBitmap failed") + ErrVerifyHeaderFail = WrapStagedSyncError("verify header failed") + ErrInsertChainFail = WrapStagedSyncError("insert to chain failed") + ErrZeroBlockResponse = WrapStagedSyncError("zero block number response from remote nodes") + ErrEmptyWhitelist = WrapStagedSyncError("empty white list") + ErrWrongGetBlockNumberType = WrapStagedSyncError("wrong type of getBlockNumber interface") + ErrSaveBlocksToDbFailed = WrapStagedSyncError("saving downloaded blocks to db failed") ) // WrapStagedSyncError wraps errors for staged sync and returns error object diff --git a/api/service/stagedstreamsync/helpers.go b/api/service/stagedstreamsync/helpers.go index 9e986a538b..75e504214f 100644 --- a/api/service/stagedstreamsync/helpers.go +++ b/api/service/stagedstreamsync/helpers.go @@ -8,7 +8,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/harmony/core/types" sttypes "github.com/harmony-one/harmony/p2p/stream/types" - "github.com/pkg/errors" ) func marshalData(blockNumber uint64) []byte { @@ -61,11 +60,11 @@ func computeBlockNumberByMaxVote(votes map[sttypes.StreamID]uint64) uint64 { func checkGetBlockByHashesResult(blocks []*types.Block, hashes []common.Hash) error { if len(blocks) != len(hashes) { - return errors.New("unexpected number of getBlocksByHashes result") + return ErrUnexpectedNumberOfBlockHashes } for i, block := range blocks { if block == nil { - return errors.New("nil block found") + return ErrNilBlock } if block.Hash() != hashes[i] { return fmt.Errorf("unexpected block hash: %x / %x", block.Hash(), hashes[i]) diff --git a/api/service/stagedstreamsync/metric.go b/api/service/stagedstreamsync/metric.go index 1bc11a227f..9437cc09e3 100644 --- a/api/service/stagedstreamsync/metric.go +++ b/api/service/stagedstreamsync/metric.go @@ -23,7 +23,7 @@ var ( consensusTriggeredDownloadCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "staged_stream_sync", Name: "consensus_trigger", Help: "number of times consensus triggered download task", }, @@ -33,7 +33,7 @@ var ( longRangeSyncedBlockCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "staged_stream_sync", Name: "num_blocks_synced_long_range", Help: "number of blocks synced in long range sync", }, @@ -43,7 +43,7 @@ var ( longRangeFailInsertedBlockCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "staged_stream_sync", Name: "num_blocks_failed_long_range", Help: "number of blocks failed to insert into change in long range sync", }, @@ -53,7 +53,7 @@ var ( numShortRangeCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "staged_stream_sync", Name: "num_short_range", Help: "number of short range sync is triggered", }, @@ -63,7 +63,7 @@ var ( numFailedDownloadCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "staged_stream_sync", Name: "failed_download", Help: "number of downloading is failed", }, @@ -73,7 +73,7 @@ var ( numBlocksInsertedShortRangeHistogramVec = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "staged_stream_sync", Name: "num_blocks_inserted_short_range", Help: "number of blocks inserted for each short range sync", // Buckets: 0, 1, 2, 4, +INF (capped at 10) @@ -85,7 +85,7 @@ var ( numBlocksInsertedBeaconHelperCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "staged_stream_sync", Name: "num_blocks_inserted_beacon_helper", Help: "number of blocks inserted from beacon helper", }, diff --git a/api/service/stagedstreamsync/short_range_helper.go b/api/service/stagedstreamsync/short_range_helper.go index f8dd2d28ea..210d1aa0c8 100644 --- a/api/service/stagedstreamsync/short_range_helper.go +++ b/api/service/stagedstreamsync/short_range_helper.go @@ -134,7 +134,7 @@ func (sh *srHelper) getBlocksByHashes(hashes []common.Hash, whitelist []sttypes. func (sh *srHelper) checkPrerequisites() error { if sh.syncProtocol.NumStreams() < sh.config.Concurrency { - return errors.New("not enough streams") + return ErrNotEnoughStreams } return nil } @@ -166,12 +166,11 @@ func (sh *srHelper) doGetBlockHashesRequest(bns []uint64) ([]common.Hash, sttype return nil, stid, err } if len(hashes) != len(bns) { - err := errors.New("unexpected get block hashes result delivered") - sh.logger.Warn().Err(err). + sh.logger.Warn().Err(ErrUnexpectedBlockHashes). Str("stream", string(stid)). Msg(WrapStagedSyncMsg("failed to doGetBlockHashesRequest")) sh.syncProtocol.StreamFailed(stid, "unexpected get block hashes result delivered") - return nil, stid, err + return nil, stid, ErrUnexpectedBlockHashes } return hashes, stid, nil } @@ -182,7 +181,9 @@ func (sh *srHelper) doGetBlocksByNumbersRequest(bns []uint64) ([]*types.Block, s blocks, stid, err := sh.syncProtocol.GetBlocksByNumber(ctx, bns) if err != nil { - sh.logger.Warn().Err(err).Str("stream", string(stid)).Msg(WrapStagedSyncMsg("failed to doGetBlockHashesRequest")) + sh.logger.Warn().Err(err). + Str("stream", string(stid)). + Msg(WrapStagedSyncMsg("failed to doGetBlockHashesRequest")) return nil, stid, err } return blocks, stid, nil diff --git a/api/service/stagedstreamsync/stage.go b/api/service/stagedstreamsync/stage.go index 832cf4e3c7..255560a0ff 100644 --- a/api/service/stagedstreamsync/stage.go +++ b/api/service/stagedstreamsync/stage.go @@ -2,7 +2,6 @@ package stagedstreamsync import ( "context" - "errors" "github.com/ethereum/go-ethereum/common" sttypes "github.com/harmony-one/harmony/p2p/stream/types" @@ -48,9 +47,6 @@ type Stage struct { Disabled bool } -var ErrStopped = errors.New("stopped") -var ErrRevert = errors.New("unwound") - // StageState is the state of the stage. type StageState struct { state *StagedStreamSync diff --git a/api/service/stagedstreamsync/stage_bodies.go b/api/service/stagedstreamsync/stage_bodies.go index 1f122b0d9c..1fbfcbc2e6 100644 --- a/api/service/stagedstreamsync/stage_bodies.go +++ b/api/service/stagedstreamsync/stage_bodies.go @@ -170,7 +170,7 @@ func (b *StageBodies) runBlockWorkerLoop(gbm *blockDownloadManager, wg *sync.Wai gbm.HandleRequestError(batch, err, stid) } else { if err = b.saveBlocks(gbm.tx, batch, blockBytes, sigBytes, loopID, stid); err != nil { - panic("[STAGED_STREAM_SYNC] saving downloaded blocks to db failed.") + panic(ErrSaveBlocksToDbFailed) } gbm.HandleRequestResult(batch, blockBytes, sigBytes, loopID, stid) if b.configs.logProgress { diff --git a/api/service/stagedstreamsync/syncing.go b/api/service/stagedstreamsync/syncing.go index 046ba6843b..b793151c63 100644 --- a/api/service/stagedstreamsync/syncing.go +++ b/api/service/stagedstreamsync/syncing.go @@ -314,7 +314,7 @@ func (s *StagedStreamSync) estimateCurrentNumber() (uint64, error) { return 0, s.ctx.Err() default: } - return 0, errors.New("zero block number response from remote nodes") + return 0, ErrZeroBlockResponse } bn := computeBlockNumberByMaxVote(cnResults) return bn, nil diff --git a/api/service/stagedstreamsync/types.go b/api/service/stagedstreamsync/types.go index 00cfba79a2..6d6326452e 100644 --- a/api/service/stagedstreamsync/types.go +++ b/api/service/stagedstreamsync/types.go @@ -270,7 +270,7 @@ func (q priorityQueue) Swap(i, j int) { func (q *priorityQueue) Push(x interface{}) { item, ok := x.(bnPrioritizedItem) if !ok { - panic("wrong type of getBlockNumber interface") + panic(ErrWrongGetBlockNumberType) } *q = append(*q, item) } diff --git a/hmy/downloader/metric.go b/hmy/downloader/metric.go index 9338fd1304..2995db7b31 100644 --- a/hmy/downloader/metric.go +++ b/hmy/downloader/metric.go @@ -23,7 +23,7 @@ var ( consensusTriggeredDownloadCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "consensus_trigger", Help: "number of times consensus triggered download task", }, @@ -33,7 +33,7 @@ var ( longRangeSyncedBlockCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "num_blocks_synced_long_range", Help: "number of blocks synced in long range sync", }, @@ -43,7 +43,7 @@ var ( longRangeFailInsertedBlockCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "num_blocks_failed_long_range", Help: "number of blocks failed to insert into change in long range sync", }, @@ -53,7 +53,7 @@ var ( numShortRangeCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "num_short_range", Help: "number of short range sync is triggered", }, @@ -63,7 +63,7 @@ var ( numFailedDownloadCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "failed_download", Help: "number of downloading is failed", }, @@ -73,7 +73,7 @@ var ( numBlocksInsertedShortRangeHistogramVec = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "num_blocks_inserted_short_range", Help: "number of blocks inserted for each short range sync", // Buckets: 0, 1, 2, 4, +INF (capped at 10) @@ -85,7 +85,7 @@ var ( numBlocksInsertedBeaconHelperCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "num_blocks_inserted_beacon_helper", Help: "number of blocks inserted from beacon helper", }, From 7280edeb1440e8d6de148c9edb2795ce35933ece Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Mon, 2 Jan 2023 22:48:42 +0800 Subject: [PATCH 179/420] refactor p2p host creation --- p2p/host.go | 56 +++++++++++++++++-- .../common/streammanager/streammanager.go | 10 +++- p2p/stream/types/utils.go | 4 +- 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/p2p/host.go b/p2p/host.go index a2326c8125..f64a0cd624 100644 --- a/p2p/host.go +++ b/p2p/host.go @@ -12,6 +12,9 @@ import ( "time" "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/routing" + dht "github.com/libp2p/go-libp2p-kad-dht" libp2p_pubsub "github.com/libp2p/go-libp2p-pubsub" libp2p_crypto "github.com/libp2p/go-libp2p/core/crypto" libp2p_host "github.com/libp2p/go-libp2p/core/host" @@ -19,6 +22,10 @@ import ( libp2p_peer "github.com/libp2p/go-libp2p/core/peer" libp2p_peerstore "github.com/libp2p/go-libp2p/core/peerstore" "github.com/libp2p/go-libp2p/core/protocol" + "github.com/libp2p/go-libp2p/p2p/net/connmgr" + + "github.com/libp2p/go-libp2p/p2p/security/noise" + libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls" ma "github.com/multiformats/go-multiaddr" "github.com/pkg/errors" "github.com/rs/zerolog" @@ -111,19 +118,56 @@ func NewHost(cfg HostConfig) (Host, error) { self = cfg.Self key = cfg.BLSKey ) - listenAddr, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%s", self.IP, self.Port)) + // listenAddr, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%s", self.IP, self.Port)) + // if err != nil { + // return nil, errors.Wrapf(err, + // "cannot create listen multiaddr from port %#v", self.Port) + // } + + addr := fmt.Sprintf("/ip4/%s/tcp/%s", self.IP, self.Port) + listenAddr := libp2p.ListenAddrStrings( + addr, // regular tcp connections + addr+"/quic", // a UDP endpoint for the QUIC transport + ) + + ctx, cancel := context.WithCancel(context.Background()) + connmgr, err := connmgr.NewConnManager( + int(cfg.MaxPeers), // Lowwater + int(cfg.MaxPeers)*cfg.MaxConnPerIP, // HighWater, + connmgr.WithGracePeriod(time.Minute), + ) if err != nil { - return nil, errors.Wrapf(err, - "cannot create listen multiaddr from port %#v", self.Port) + cancel() + return nil, err } + var idht *dht.IpfsDHT - ctx, cancel := context.WithCancel(context.Background()) p2pHost, err := libp2p.New( - libp2p.ListenAddrs(listenAddr), + listenAddr, libp2p.Identity(key), + // support TLS connections + libp2p.Security(libp2ptls.ID, libp2ptls.New), + // support noise connections + libp2p.Security(noise.ID, noise.New), + // support any other default transports (TCP) + libp2p.DefaultTransports, + // Let's prevent our peer from having too many + // connections by attaching a connection manager. + libp2p.ConnectionManager(connmgr), + // Attempt to open ports using uPNP for NATed hosts. + libp2p.NATPortMap(), + libp2p.Routing(func(h host.Host) (routing.PeerRouting, error) { + idht, err = dht.New(ctx, h) + return idht, err + }), + // to help other peers to figure out if they are behind + // NATs, launch the server-side of AutoNAT too (AutoRelay + // already runs the client) + // This service is highly rate-limited and should not cause any + // performance issues. libp2p.EnableNATService(), - libp2p.ForceReachabilityPublic(), libp2p.BandwidthReporter(newCounter()), + libp2p.ForceReachabilityPublic(), // prevent dialing of public addresses libp2p.ConnectionGater(NewGater(cfg.DisablePrivateIPScan)), ) diff --git a/p2p/stream/common/streammanager/streammanager.go b/p2p/stream/common/streammanager/streammanager.go index a341184981..fe4505029c 100644 --- a/p2p/stream/common/streammanager/streammanager.go +++ b/p2p/stream/common/streammanager/streammanager.go @@ -419,9 +419,13 @@ func (ss *streamSet) addStream(st sttypes.Stream) { ss.lock.Lock() defer ss.lock.Unlock() - ss.streams[st.ID()] = st - spec, _ := st.ProtoSpec() - ss.numByProto[spec]++ + if spec, err := st.ProtoSpec(); err != nil { + return + } else { + ss.streams[st.ID()] = st + ss.numByProto[spec]++ + } + } func (ss *streamSet) deleteStream(st sttypes.Stream) { diff --git a/p2p/stream/types/utils.go b/p2p/stream/types/utils.go index 86eb882183..c27d95d60f 100644 --- a/p2p/stream/types/utils.go +++ b/p2p/stream/types/utils.go @@ -28,6 +28,7 @@ const ( // ProtoID is the protocol id for streaming, an alias of libp2p stream protocol ID。 // The stream protocol ID is composed of following components: +// ex: harmony/sync/partner/0/1.0.0/1 // 1. Service - Currently, only sync service is supported. // 2. NetworkType - mainnet, testnet, stn, e.t.c. // 3. ShardID - shard ID of the current protocol. @@ -37,8 +38,7 @@ type ProtoID libp2p_proto.ID // ProtoSpec is the un-serialized stream proto id specification // TODO: move this to service wise module since different protocol might have different -// -// protoID information +// protoID information type ProtoSpec struct { Service string NetworkType nodeconfig.NetworkType From e6452ff64b8c1b6240c84b1333b492eb88cc2773 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Tue, 3 Jan 2023 21:35:04 +0800 Subject: [PATCH 180/420] fix initsync and host creation --- api/service/stagedstreamsync/downloader.go | 15 ++++++++------- api/service/stagedstreamsync/stage_heads.go | 2 -- p2p/host.go | 16 +++++++--------- p2p/stream/protocols/sync/protocol.go | 2 +- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go index bb3b8c82eb..28c43c3e87 100644 --- a/api/service/stagedstreamsync/downloader.go +++ b/api/service/stagedstreamsync/downloader.go @@ -186,7 +186,7 @@ func (d *Downloader) waitForBootFinish() { func (d *Downloader) loop() { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() - initSync := d.bc.ShardID() != shard.BeaconChainShardID + initSync := true trigger := func() { select { case d.downloadC <- struct{}{}: @@ -228,11 +228,13 @@ func (d *Downloader) loop() { time.Sleep(1 * time.Second) continue } - d.logger.Info().Int("block added", addedBN). - Uint64("current height", d.bc.CurrentBlock().NumberU64()). - Bool("initSync", initSync). - Uint32("shard", d.bc.ShardID()). - Msg(WrapStagedSyncMsg("sync finished")) + if initSync { + d.logger.Info().Int("block added", addedBN). + Uint64("current height", d.bc.CurrentBlock().NumberU64()). + Bool("initSync", initSync). + Uint32("shard", d.bc.ShardID()). + Msg(WrapStagedSyncMsg("sync finished")) + } if addedBN != 0 { // If block number has been changed, trigger another sync @@ -242,7 +244,6 @@ func (d *Downloader) loop() { d.bh.insertSync() } } - d.stagedSyncInstance.initSync = false initSync = false case <-d.closeC: diff --git a/api/service/stagedstreamsync/stage_heads.go b/api/service/stagedstreamsync/stage_heads.go index 1ddd285aa0..8e1531a5ee 100644 --- a/api/service/stagedstreamsync/stage_heads.go +++ b/api/service/stagedstreamsync/stage_heads.go @@ -73,8 +73,6 @@ func (heads *StageHeads) Exec(firstCycle bool, invalidBlockRevert bool, s *Stage } if currentHeight >= maxHeight { - utils.Logger().Info().Uint64("current number", currentHeight).Uint64("target number", maxHeight). - Msg(WrapStagedSyncMsg("early return of long range sync")) return nil } diff --git a/p2p/host.go b/p2p/host.go index f64a0cd624..a2bd5996b8 100644 --- a/p2p/host.go +++ b/p2p/host.go @@ -118,11 +118,6 @@ func NewHost(cfg HostConfig) (Host, error) { self = cfg.Self key = cfg.BLSKey ) - // listenAddr, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%s", self.IP, self.Port)) - // if err != nil { - // return nil, errors.Wrapf(err, - // "cannot create listen multiaddr from port %#v", self.Port) - // } addr := fmt.Sprintf("/ip4/%s/tcp/%s", self.IP, self.Port) listenAddr := libp2p.ListenAddrStrings( @@ -131,9 +126,10 @@ func NewHost(cfg HostConfig) (Host, error) { ) ctx, cancel := context.WithCancel(context.Background()) + // TODO: move low and high to configs connmgr, err := connmgr.NewConnManager( - int(cfg.MaxPeers), // Lowwater - int(cfg.MaxPeers)*cfg.MaxConnPerIP, // HighWater, + int(10), // Lowwater + int(10000)*cfg.MaxConnPerIP, // HighWater, connmgr.WithGracePeriod(time.Minute), ) if err != nil { @@ -141,7 +137,6 @@ func NewHost(cfg HostConfig) (Host, error) { return nil, err } var idht *dht.IpfsDHT - p2pHost, err := libp2p.New( listenAddr, libp2p.Identity(key), @@ -167,7 +162,10 @@ func NewHost(cfg HostConfig) (Host, error) { // performance issues. libp2p.EnableNATService(), libp2p.BandwidthReporter(newCounter()), - libp2p.ForceReachabilityPublic(), + // ForceReachabilityPublic overrides automatic reachability detection in the AutoNAT subsystem, + // forcing the local node to believe it is reachable externally. + // libp2p.ForceReachabilityPublic(), + // prevent dialing of public addresses libp2p.ConnectionGater(NewGater(cfg.DisablePrivateIPScan)), ) diff --git a/p2p/stream/protocols/sync/protocol.go b/p2p/stream/protocols/sync/protocol.go index c63e6c45a5..68f92bb8e1 100644 --- a/p2p/stream/protocols/sync/protocol.go +++ b/p2p/stream/protocols/sync/protocol.go @@ -171,7 +171,7 @@ func (p *Protocol) HandleStream(raw libp2p_network.Stream) { Msg("failed to add new stream") return } - fmt.Println("Node connected to", raw.Conn().RemotePeer().String(), "(", st.ProtoID(), ")") + fmt.Println("Connected to", raw.Conn().RemotePeer().String(), "(", st.ProtoID(), ")") st.run() } From 057a7293fb39ef26b5235a67569c621c53c5ffa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Wed, 4 Jan 2023 14:34:09 +0800 Subject: [PATCH 181/420] fix short range hash chain --- api/service/stagedstreamsync/short_range_helper.go | 10 +++------- api/service/stagedstreamsync/stage_short_range.go | 8 +------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/api/service/stagedstreamsync/short_range_helper.go b/api/service/stagedstreamsync/short_range_helper.go index 210d1aa0c8..90415c87c3 100644 --- a/api/service/stagedstreamsync/short_range_helper.go +++ b/api/service/stagedstreamsync/short_range_helper.go @@ -139,15 +139,11 @@ func (sh *srHelper) checkPrerequisites() error { return nil } -func (sh *srHelper) prepareBlockHashNumbers(curNumber uint64, count int) []uint64 { +func (sh *srHelper) prepareBlockHashNumbers(curNumber uint64) []uint64 { - n := count - if count > BlockHashesPerRequest { - n = BlockHashesPerRequest - } - res := make([]uint64, 0, n) + res := make([]uint64, 0, BlockHashesPerRequest) - for bn := curNumber + 1; bn <= curNumber+uint64(n); bn++ { + for bn := curNumber + 1; bn <= curNumber+uint64(BlockHashesPerRequest); bn++ { res = append(res, bn) } return res diff --git a/api/service/stagedstreamsync/stage_short_range.go b/api/service/stagedstreamsync/stage_short_range.go index de83beafa3..cfe29f39e1 100644 --- a/api/service/stagedstreamsync/stage_short_range.go +++ b/api/service/stagedstreamsync/stage_short_range.go @@ -54,11 +54,6 @@ func (sr *StageShortRange) Exec(firstCycle bool, invalidBlockRevert bool, s *Sta return nil } - curBN := sr.configs.bc.CurrentBlock().NumberU64() - if curBN >= s.state.status.targetBN { - return nil - } - // do short range sync n, err := sr.doShortRangeSync(s) s.state.inserted = n @@ -109,8 +104,7 @@ func (sr *StageShortRange) doShortRangeSync(s *StageState) (int, error) { return 0, errors.Wrap(err, "prerequisite") } curBN := sr.configs.bc.CurrentBlock().NumberU64() - blkCount := int(s.state.status.targetBN) - int(curBN) - blkNums := sh.prepareBlockHashNumbers(curBN, blkCount) + blkNums := sh.prepareBlockHashNumbers(curBN) hashChain, whitelist, err := sh.getHashChain(blkNums) if err != nil { return 0, errors.Wrap(err, "getHashChain") From 8bd82bf82a7833b66d649d3b802f6ca720b02086 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Thu, 5 Jan 2023 15:34:43 +0800 Subject: [PATCH 182/420] fix beacon node detection for p2p protocol --- api/service/stagedstreamsync/downloader.go | 13 +++++++------ api/service/stagedstreamsync/downloaders.go | 4 ++-- hmy/downloader/downloader.go | 13 +++++++------ hmy/downloader/downloaders.go | 4 ++-- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go index 28c43c3e87..a7b7402c21 100644 --- a/api/service/stagedstreamsync/downloader.go +++ b/api/service/stagedstreamsync/downloader.go @@ -36,15 +36,16 @@ type ( ) // NewDownloader creates a new downloader -func NewDownloader(host p2p.Host, bc core.BlockChain, config Config) *Downloader { +func NewDownloader(host p2p.Host, bc core.BlockChain, isBeaconNode bool, config Config) *Downloader { config.fixValues() sp := sync.NewProtocol(sync.Config{ - Chain: bc, - Host: host.GetP2PHost(), - Discovery: host.GetDiscovery(), - ShardID: nodeconfig.ShardID(bc.ShardID()), - Network: config.Network, + Chain: bc, + Host: host.GetP2PHost(), + Discovery: host.GetDiscovery(), + ShardID: nodeconfig.ShardID(bc.ShardID()), + Network: config.Network, + BeaconNode: isBeaconNode, SmSoftLowCap: config.SmSoftLowCap, SmHardLowCap: config.SmHardLowCap, diff --git a/api/service/stagedstreamsync/downloaders.go b/api/service/stagedstreamsync/downloaders.go index 7b5881f100..0e79c79631 100644 --- a/api/service/stagedstreamsync/downloaders.go +++ b/api/service/stagedstreamsync/downloaders.go @@ -17,7 +17,7 @@ type Downloaders struct { // NewDownloaders creates Downloaders for sync of multiple blockchains func NewDownloaders(host p2p.Host, bcs []core.BlockChain, config Config) *Downloaders { ds := make(map[uint32]*Downloader) - + isBeaconNode := len(bcs) == 1 for _, bc := range bcs { if bc == nil { continue @@ -25,7 +25,7 @@ func NewDownloaders(host p2p.Host, bcs []core.BlockChain, config Config) *Downlo if _, ok := ds[bc.ShardID()]; ok { continue } - ds[bc.ShardID()] = NewDownloader(host, bc, config) + ds[bc.ShardID()] = NewDownloader(host, bc, isBeaconNode, config) } return &Downloaders{ ds: ds, diff --git a/hmy/downloader/downloader.go b/hmy/downloader/downloader.go index f5ab8580b7..01ec242abb 100644 --- a/hmy/downloader/downloader.go +++ b/hmy/downloader/downloader.go @@ -44,15 +44,16 @@ type ( ) // NewDownloader creates a new downloader -func NewDownloader(host p2p.Host, bc core.BlockChain, config Config) *Downloader { +func NewDownloader(host p2p.Host, bc core.BlockChain, isBeaconNode bool, config Config) *Downloader { config.fixValues() sp := sync.NewProtocol(sync.Config{ - Chain: bc, - Host: host.GetP2PHost(), - Discovery: host.GetDiscovery(), - ShardID: nodeconfig.ShardID(bc.ShardID()), - Network: config.Network, + Chain: bc, + Host: host.GetP2PHost(), + Discovery: host.GetDiscovery(), + ShardID: nodeconfig.ShardID(bc.ShardID()), + Network: config.Network, + BeaconNode: isBeaconNode, SmSoftLowCap: config.SmSoftLowCap, SmHardLowCap: config.SmHardLowCap, diff --git a/hmy/downloader/downloaders.go b/hmy/downloader/downloaders.go index a1cbfa5a82..528e47e84f 100644 --- a/hmy/downloader/downloaders.go +++ b/hmy/downloader/downloaders.go @@ -17,7 +17,7 @@ type Downloaders struct { // NewDownloaders creates Downloaders for sync of multiple blockchains func NewDownloaders(host p2p.Host, bcs []core.BlockChain, config Config) *Downloaders { ds := make(map[uint32]*Downloader) - + isBeaconNode := len(bcs) == 1 for _, bc := range bcs { if bc == nil { continue @@ -25,7 +25,7 @@ func NewDownloaders(host p2p.Host, bcs []core.BlockChain, config Config) *Downlo if _, ok := ds[bc.ShardID()]; ok { continue } - ds[bc.ShardID()] = NewDownloader(host, bc, config) + ds[bc.ShardID()] = NewDownloader(host, bc, isBeaconNode, config) } return &Downloaders{ ds: ds, From 390c9f99c1af3dae80a46f949e9d0dd61a857cdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Fri, 6 Jan 2023 09:02:40 +0800 Subject: [PATCH 183/420] refactor stream peer cooldown and fix protocol beacon node field --- p2p/stream/common/streammanager/cooldown.go | 7 ++++++- .../common/streammanager/streammanager.go | 4 ++++ p2p/stream/protocols/sync/protocol.go | 20 +++++++------------ 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/p2p/stream/common/streammanager/cooldown.go b/p2p/stream/common/streammanager/cooldown.go index d8b58346c0..f22f4956f0 100644 --- a/p2p/stream/common/streammanager/cooldown.go +++ b/p2p/stream/common/streammanager/cooldown.go @@ -30,11 +30,16 @@ func newCoolDownCache() *coolDownCache { func (cache *coolDownCache) Has(id peer.ID) bool { cache.mu.Lock() defer cache.mu.Unlock() + has := cache.timeCache.Has(string(id)) + return has +} + +// Add adds the peer ID to the cache +func (cache *coolDownCache) Add(id peer.ID) { has := cache.timeCache.Has(string(id)) if !has { cache.timeCache.Add(string(id)) } - return has } // Reset the cool down cache diff --git a/p2p/stream/common/streammanager/streammanager.go b/p2p/stream/common/streammanager/streammanager.go index fe4505029c..60d3a55213 100644 --- a/p2p/stream/common/streammanager/streammanager.go +++ b/p2p/stream/common/streammanager/streammanager.go @@ -315,12 +315,16 @@ func (sm *streamManager) discoverAndSetupStream(discCtx context.Context) (int, e // If the peer has the same ID and was just connected, skip. continue } + if _, ok := sm.streams.get(sttypes.StreamID(peer.ID)); ok { + continue + } discoveredPeersCounterVec.With(prometheus.Labels{"topic": string(sm.myProtoID)}).Inc() connecting += 1 go func(pid libp2p_peer.ID) { // The ctx here is using the module context instead of discover context err := sm.setupStreamWithPeer(sm.ctx, pid) if err != nil { + sm.coolDownCache.Add(peer.ID) sm.logger.Warn().Err(err).Str("peerID", string(pid)).Msg("failed to setup stream with peer") return } diff --git a/p2p/stream/protocols/sync/protocol.go b/p2p/stream/protocols/sync/protocol.go index 68f92bb8e1..7b5a1ff2d2 100644 --- a/p2p/stream/protocols/sync/protocol.go +++ b/p2p/stream/protocols/sync/protocol.go @@ -8,7 +8,6 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/harmony-one/harmony/consensus/engine" - "github.com/harmony-one/harmony/core" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" "github.com/harmony-one/harmony/internal/utils" @@ -17,7 +16,6 @@ import ( "github.com/harmony-one/harmony/p2p/stream/common/requestmanager" "github.com/harmony-one/harmony/p2p/stream/common/streammanager" sttypes "github.com/harmony-one/harmony/p2p/stream/types" - "github.com/harmony-one/harmony/shard" "github.com/hashicorp/go-version" libp2p_host "github.com/libp2p/go-libp2p/core/host" libp2p_network "github.com/libp2p/go-libp2p/core/network" @@ -60,12 +58,12 @@ type ( // Config is the sync protocol config Config struct { - Chain engine.ChainReader - Host libp2p_host.Host - Discovery discovery.Discovery - ShardID nodeconfig.ShardID - Network nodeconfig.NetworkType - + Chain engine.ChainReader + Host libp2p_host.Host + Discovery discovery.Discovery + ShardID nodeconfig.ShardID + Network nodeconfig.NetworkType + BeaconNode bool // stream manager config SmSoftLowCap int SmHardLowCap int @@ -78,13 +76,9 @@ type ( func NewProtocol(config Config) *Protocol { ctx, cancel := context.WithCancel(context.Background()) - isBeaconNode := config.Chain.ShardID() == shard.BeaconChainShardID - if _, ok := config.Chain.(*core.EpochChain); ok { - isBeaconNode = false - } sp := &Protocol{ chain: config.Chain, - beaconNode: isBeaconNode, + beaconNode: config.BeaconNode, disc: config.Discovery, config: config, ctx: ctx, From 905ff653cc09ff0edca0ae2b1d7a9a2f8319f377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Wed, 11 Jan 2023 22:55:15 +0800 Subject: [PATCH 184/420] refactor p2p host and routing --- api/service/stagedstreamsync/beacon_helper.go | 14 ++++---- api/service/stagedstreamsync/downloader.go | 22 +++++++----- api/service/stagedstreamsync/stage_epoch.go | 2 +- .../stagedstreamsync/stage_short_range.go | 3 +- .../stagedstreamsync/staged_stream_sync.go | 29 +++++++++------- api/service/stagedstreamsync/syncing.go | 2 ++ cmd/harmony/main.go | 8 ++--- p2p/discovery/discovery.go | 14 ++------ p2p/discovery/option.go | 8 +++-- p2p/host.go | 31 ++++++++++++----- .../common/streammanager/streammanager.go | 9 +++-- p2p/stream/protocols/sync/protocol.go | 34 +++++++++++++++++-- p2p/stream/types/interface.go | 2 ++ 13 files changed, 115 insertions(+), 63 deletions(-) diff --git a/api/service/stagedstreamsync/beacon_helper.go b/api/service/stagedstreamsync/beacon_helper.go index 2f684e0e27..a996f368bf 100644 --- a/api/service/stagedstreamsync/beacon_helper.go +++ b/api/service/stagedstreamsync/beacon_helper.go @@ -76,17 +76,19 @@ func (bh *beaconHelper) loop() { case it := <-bh.insertC: inserted, bn, err := bh.insertLastMileBlocks() - numBlocksInsertedBeaconHelperCounter.Add(float64(inserted)) if err != nil { bh.logger.Error().Err(err). Msg(WrapStagedSyncMsg("insert last mile blocks error")) + close(it.doneC) continue } - bh.logger.Info().Int("inserted", inserted). - Uint64("end height", bn). - Uint32("shard", bh.bc.ShardID()). - Msg(WrapStagedSyncMsg("insert last mile blocks")) - + if inserted > 0 { + numBlocksInsertedBeaconHelperCounter.Add(float64(inserted)) + bh.logger.Info().Int("inserted", inserted). + Uint64("end height", bn). + Uint32("shard", bh.bc.ShardID()). + Msg(WrapStagedSyncMsg("insert last mile blocks")) + } close(it.doneC) case <-bh.closeC: diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go index a7b7402c21..f53a5c41a8 100644 --- a/api/service/stagedstreamsync/downloader.go +++ b/api/service/stagedstreamsync/downloader.go @@ -24,6 +24,7 @@ type ( syncProtocol syncProtocol bh *beaconHelper stagedSyncInstance *StagedStreamSync + isBeaconNode bool downloadC chan struct{} closeC chan struct{} @@ -67,7 +68,7 @@ func NewDownloader(host p2p.Host, bc core.BlockChain, isBeaconNode bool, config ctx, cancel := context.WithCancel(context.Background()) //TODO: use mem db should be in config file - stagedSyncInstance, err := CreateStagedSync(ctx, bc, false, sp, config, logger, config.LogProgress) + stagedSyncInstance, err := CreateStagedSync(ctx, bc, false, isBeaconNode, sp, config, logger, config.LogProgress) if err != nil { cancel() return nil @@ -78,6 +79,7 @@ func NewDownloader(host p2p.Host, bc core.BlockChain, isBeaconNode bool, config syncProtocol: sp, bh: bh, stagedSyncInstance: stagedSyncInstance, + isBeaconNode: isBeaconNode, downloadC: make(chan struct{}), closeC: make(chan struct{}), @@ -187,7 +189,11 @@ func (d *Downloader) waitForBootFinish() { func (d *Downloader) loop() { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() - initSync := true + // for shard chain and beacon chain node, first we start with initSync=true to + // make sure it goes through the long range sync first. + // for epoch chain we do only need to go through epoch sync process + initSync := d.isBeaconNode || d.bc.ShardID() != shard.BeaconChainShardID + trigger := func() { select { case d.downloadC <- struct{}{}: @@ -217,7 +223,7 @@ func (d *Downloader) loop() { } } - // If error happens, sleep 5 seconds and retry + // If any error happens, sleep 5 seconds and retry d.logger.Error(). Err(err). Bool("initSync", initSync). @@ -227,7 +233,7 @@ func (d *Downloader) loop() { trigger() }() time.Sleep(1 * time.Second) - continue + break } if initSync { d.logger.Info().Int("block added", addedBN). @@ -239,11 +245,11 @@ func (d *Downloader) loop() { if addedBN != 0 { // If block number has been changed, trigger another sync - // and try to add last mile from pub-sub (blocking) go trigger() - if d.bh != nil { - d.bh.insertSync() - } + } + // try to add last mile from pub-sub (blocking) + if d.bh != nil { + d.bh.insertSync() } initSync = false diff --git a/api/service/stagedstreamsync/stage_epoch.go b/api/service/stagedstreamsync/stage_epoch.go index 6320d82175..77dc57bfdd 100644 --- a/api/service/stagedstreamsync/stage_epoch.go +++ b/api/service/stagedstreamsync/stage_epoch.go @@ -49,7 +49,7 @@ func (sr *StageEpoch) Exec(firstCycle bool, invalidBlockRevert bool, s *StageSta return nil } - if _, ok := sr.configs.bc.(*core.EpochChain); !ok { + if sr.configs.bc.ShardID() != shard.BeaconChainShardID || s.state.isBeaconNode { return nil } diff --git a/api/service/stagedstreamsync/stage_short_range.go b/api/service/stagedstreamsync/stage_short_range.go index cfe29f39e1..75f51ee1e7 100644 --- a/api/service/stagedstreamsync/stage_short_range.go +++ b/api/service/stagedstreamsync/stage_short_range.go @@ -6,6 +6,7 @@ import ( "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/internal/utils" sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/harmony-one/harmony/shard" "github.com/ledgerwatch/erigon-lib/kv" "github.com/pkg/errors" ) @@ -50,7 +51,7 @@ func (sr *StageShortRange) Exec(firstCycle bool, invalidBlockRevert bool, s *Sta return nil } - if _, ok := sr.configs.bc.(*core.EpochChain); ok { + if sr.configs.bc.ShardID() == shard.BeaconChainShardID && !s.state.isBeaconNode { return nil } diff --git a/api/service/stagedstreamsync/staged_stream_sync.go b/api/service/stagedstreamsync/staged_stream_sync.go index 9603f5b06b..e73edd622f 100644 --- a/api/service/stagedstreamsync/staged_stream_sync.go +++ b/api/service/stagedstreamsync/staged_stream_sync.go @@ -54,19 +54,20 @@ func (ib *InvalidBlock) addBadStream(bsID sttypes.StreamID) { } type StagedStreamSync struct { - ctx context.Context - bc core.BlockChain - isBeacon bool - isExplorer bool - db kv.RwDB - protocol syncProtocol - gbm *blockDownloadManager // initialized when finished get block number - inserted int - config Config - logger zerolog.Logger - status *status //TODO: merge this with currentSyncCycle - initSync bool // if sets to true, node start long range syncing - UseMemDB bool + ctx context.Context + bc core.BlockChain + isBeacon bool + isExplorer bool + db kv.RwDB + protocol syncProtocol + isBeaconNode bool + gbm *blockDownloadManager // initialized when finished get block number + inserted int + config Config + logger zerolog.Logger + status *status //TODO: merge this with currentSyncCycle + initSync bool // if sets to true, node start long range syncing + UseMemDB bool revertPoint *uint64 // used to run stages prevRevertPoint *uint64 // used to get value from outside of staged sync after cycle (for example to notify RPCDaemon) @@ -254,6 +255,7 @@ func New(ctx context.Context, stagesList []*Stage, isBeacon bool, protocol syncProtocol, + isBeaconNode bool, useMemDB bool, config Config, logger zerolog.Logger, @@ -291,6 +293,7 @@ func New(ctx context.Context, isBeacon: isBeacon, db: db, protocol: protocol, + isBeaconNode: isBeaconNode, gbm: nil, status: &status, inserted: 0, diff --git a/api/service/stagedstreamsync/syncing.go b/api/service/stagedstreamsync/syncing.go index b793151c63..de9b884816 100644 --- a/api/service/stagedstreamsync/syncing.go +++ b/api/service/stagedstreamsync/syncing.go @@ -38,6 +38,7 @@ var Buckets = []string{ func CreateStagedSync(ctx context.Context, bc core.BlockChain, UseMemDB bool, + isBeaconNode bool, protocol syncProtocol, config Config, logger zerolog.Logger, @@ -87,6 +88,7 @@ func CreateStagedSync(ctx context.Context, stages, isBeacon, protocol, + isBeaconNode, UseMemDB, config, logger, diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 6c83636673..6ab3ed1024 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -878,8 +878,8 @@ func setupPrometheusService(node *node.Node, hc harmonyconfig.HarmonyConfig, sid func setupSyncService(node *node.Node, host p2p.Host, hc harmonyconfig.HarmonyConfig) { blockchains := []core.BlockChain{node.Blockchain()} - if !node.IsRunningBeaconChain() { - blockchains = append(blockchains, node.Beaconchain()) + if node.Blockchain().ShardID() != shard.BeaconChainShardID { + blockchains = append(blockchains, node.EpochChain()) } dConfig := downloader.Config{ @@ -913,8 +913,8 @@ func setupSyncService(node *node.Node, host p2p.Host, hc harmonyconfig.HarmonyCo func setupStagedSyncService(node *node.Node, host p2p.Host, hc harmonyconfig.HarmonyConfig) { blockchains := []core.BlockChain{node.Blockchain()} - if !node.IsRunningBeaconChain() { - blockchains = append(blockchains, node.Beaconchain()) + if node.Blockchain().ShardID() != shard.BeaconChainShardID { + blockchains = append(blockchains, node.EpochChain()) } sConfig := stagedstreamsync.Config{ diff --git a/p2p/discovery/discovery.go b/p2p/discovery/discovery.go index fb9591c267..53372edd67 100644 --- a/p2p/discovery/discovery.go +++ b/p2p/discovery/discovery.go @@ -5,6 +5,7 @@ import ( "time" "github.com/harmony-one/harmony/internal/utils" + dht "github.com/libp2p/go-libp2p-kad-dht" libp2p_dht "github.com/libp2p/go-libp2p-kad-dht" "github.com/libp2p/go-libp2p/core/discovery" libp2p_host "github.com/libp2p/go-libp2p/core/host" @@ -37,19 +38,8 @@ type dhtDiscovery struct { } // NewDHTDiscovery creates a new dhtDiscovery that implements Discovery interface. -func NewDHTDiscovery(host libp2p_host.Host, opt DHTConfig) (Discovery, error) { - opts, err := opt.getLibp2pRawOptions() - if err != nil { - return nil, err - } - ctx, cancel := context.WithCancel(context.Background()) - dht, err := libp2p_dht.New(ctx, host, opts...) - if err != nil { - cancel() - return nil, err - } +func NewDHTDiscovery(ctx context.Context, cancel context.CancelFunc, host libp2p_host.Host, dht *dht.IpfsDHT, opt DHTConfig) (Discovery, error) { d := libp2p_dis.NewRoutingDiscovery(dht) - logger := utils.Logger().With().Str("module", "discovery").Logger() return &dhtDiscovery{ dht: dht, diff --git a/p2p/discovery/option.go b/p2p/discovery/option.go index 0afe6b8a2f..ce42592703 100644 --- a/p2p/discovery/option.go +++ b/p2p/discovery/option.go @@ -5,6 +5,7 @@ import ( p2ptypes "github.com/harmony-one/harmony/p2p/types" badger "github.com/ipfs/go-ds-badger" + dht "github.com/libp2p/go-libp2p-kad-dht" libp2p_dht "github.com/libp2p/go-libp2p-kad-dht" ) @@ -14,10 +15,11 @@ type DHTConfig struct { BootNodes []string DataStoreFile *string // File path to store DHT data. Shall be only used for bootstrap nodes. DiscConcurrency int + DHT *dht.IpfsDHT } -// getLibp2pRawOptions get the raw libp2p options as a slice. -func (opt DHTConfig) getLibp2pRawOptions() ([]libp2p_dht.Option, error) { +// GetLibp2pRawOptions get the raw libp2p options as a slice. +func (opt DHTConfig) GetLibp2pRawOptions() ([]libp2p_dht.Option, error) { var opts []libp2p_dht.Option bootOption, err := getBootstrapOption(opt.BootNodes) @@ -40,6 +42,8 @@ func (opt DHTConfig) getLibp2pRawOptions() ([]libp2p_dht.Option, error) { opts = append(opts, libp2p_dht.Concurrency(opt.DiscConcurrency)) } + opts = append(opts, libp2p_dht.DisableAutoRefresh()) + return opts, nil } diff --git a/p2p/host.go b/p2p/host.go index a2bd5996b8..fcd2ea3b4d 100644 --- a/p2p/host.go +++ b/p2p/host.go @@ -128,7 +128,7 @@ func NewHost(cfg HostConfig) (Host, error) { ctx, cancel := context.WithCancel(context.Background()) // TODO: move low and high to configs connmgr, err := connmgr.NewConnManager( - int(10), // Lowwater + int(10), // LowWater int(10000)*cfg.MaxConnPerIP, // HighWater, connmgr.WithGracePeriod(time.Minute), ) @@ -137,6 +137,7 @@ func NewHost(cfg HostConfig) (Host, error) { return nil, err } var idht *dht.IpfsDHT + var opt discovery.DHTConfig p2pHost, err := libp2p.New( listenAddr, libp2p.Identity(key), @@ -152,33 +153,41 @@ func NewHost(cfg HostConfig) (Host, error) { // Attempt to open ports using uPNP for NATed hosts. libp2p.NATPortMap(), libp2p.Routing(func(h host.Host) (routing.PeerRouting, error) { - idht, err = dht.New(ctx, h) + opt = discovery.DHTConfig{ + BootNodes: cfg.BootNodes, + DataStoreFile: cfg.DataStoreFile, + DiscConcurrency: cfg.DiscConcurrency, + } + opts, err := opt.GetLibp2pRawOptions() + if err != nil { + return nil, err + } + idht, err = dht.New(ctx, h, opts...) return idht, err }), + // to help other peers to figure out if they are behind // NATs, launch the server-side of AutoNAT too (AutoRelay // already runs the client) // This service is highly rate-limited and should not cause any // performance issues. libp2p.EnableNATService(), + // Bandwidth Reporter libp2p.BandwidthReporter(newCounter()), // ForceReachabilityPublic overrides automatic reachability detection in the AutoNAT subsystem, // forcing the local node to believe it is reachable externally. // libp2p.ForceReachabilityPublic(), - + // libp2p.DisableRelay(), + libp2p.EnableRelayService(), // prevent dialing of public addresses - libp2p.ConnectionGater(NewGater(cfg.DisablePrivateIPScan)), + // libp2p.ConnectionGater(NewGater(cfg.DisablePrivateIPScan)), ) if err != nil { cancel() return nil, errors.Wrapf(err, "cannot initialize libp2p host") } - disc, err := discovery.NewDHTDiscovery(p2pHost, discovery.DHTConfig{ - BootNodes: cfg.BootNodes, - DataStoreFile: cfg.DataStoreFile, - DiscConcurrency: cfg.DiscConcurrency, - }) + disc, err := discovery.NewDHTDiscovery(ctx, cancel, p2pHost, idht, opt) if err != nil { cancel() return nil, errors.Wrap(err, "cannot create DHT discovery") @@ -319,6 +328,10 @@ func (host *HostV2) AddStreamProtocol(protocols ...sttypes.Protocol) { for _, proto := range protocols { host.streamProtos = append(host.streamProtos, proto) host.h.SetStreamHandlerMatch(protocol.ID(proto.ProtoID()), proto.Match, proto.HandleStream) + // TODO: do we need to add handler match for shard proto id? + // if proto.IsBeaconNode() { + // host.h.SetStreamHandlerMatch(protocol.ID(proto.ShardProtoID()), proto.Match, proto.HandleStream) + // } } } diff --git a/p2p/stream/common/streammanager/streammanager.go b/p2p/stream/common/streammanager/streammanager.go index 60d3a55213..064cb0c2d0 100644 --- a/p2p/stream/common/streammanager/streammanager.go +++ b/p2p/stream/common/streammanager/streammanager.go @@ -73,7 +73,6 @@ func newStreamManager(pid sttypes.ProtoID, host host, pf peerFinder, handleStrea Str("protocol ID", string(pid)).Logger() protoSpec, _ := sttypes.ProtoIDToProtoSpec(pid) - fmt.Println("my peer id: ", host.ID().String()) fmt.Println("my proto id: ", pid) @@ -238,9 +237,6 @@ func (sm *streamManager) sanityCheckStream(st sttypes.Stream) error { if mySpec.ShardID != rmSpec.ShardID { return fmt.Errorf("unexpected shard ID: %v/%v", rmSpec.ShardID, mySpec.ShardID) } - if mySpec.ShardID == shard.BeaconChainShardID && !rmSpec.BeaconNode { - return fmt.Errorf("unexpected beacon node with shard ID: %v/%v", rmSpec.ShardID, mySpec.ShardID) - } return nil } @@ -311,7 +307,10 @@ func (sm *streamManager) discoverAndSetupStream(discCtx context.Context) (int, e connecting := 0 for peer := range peers { - if peer.ID == sm.host.ID() || sm.coolDownCache.Has(peer.ID) { + if peer.ID == sm.host.ID() { + continue + } + if sm.coolDownCache.Has(peer.ID) { // If the peer has the same ID and was just connected, skip. continue } diff --git a/p2p/stream/protocols/sync/protocol.go b/p2p/stream/protocols/sync/protocol.go index 7b5a1ff2d2..2a984a5818 100644 --- a/p2p/stream/protocols/sync/protocol.go +++ b/p2p/stream/protocols/sync/protocol.go @@ -16,6 +16,7 @@ import ( "github.com/harmony-one/harmony/p2p/stream/common/requestmanager" "github.com/harmony-one/harmony/p2p/stream/common/streammanager" sttypes "github.com/harmony-one/harmony/p2p/stream/types" + "github.com/harmony-one/harmony/shard" "github.com/hashicorp/go-version" libp2p_host "github.com/libp2p/go-libp2p/core/host" libp2p_network "github.com/libp2p/go-libp2p/core/network" @@ -107,7 +108,10 @@ func (p *Protocol) Start() { p.sm.Start() p.rm.Start() p.rl.Start() - go p.advertiseLoop() + // If it's not EpochChain, advertise + if p.beaconNode || p.chain.ShardID() != shard.BeaconChainShardID { + go p.advertiseLoop() + } } // Close close the protocol @@ -129,11 +133,21 @@ func (p *Protocol) ProtoID() sttypes.ProtoID { return p.protoIDByVersion(MyVersion) } +// ShardProtoID returns the ProtoID of the sync protocol for shard nodes +func (p *Protocol) ShardProtoID() sttypes.ProtoID { + return p.protoIDByVersionForShardNodes(MyVersion) +} + // Version returns the sync protocol version func (p *Protocol) Version() *version.Version { return MyVersion } +// IsBeaconNode returns true if it is a beacon chain node +func (p *Protocol) IsBeaconNode() bool { + return p.beaconNode +} + // Match checks the compatibility to the target protocol ID. func (p *Protocol) Match(targetID string) bool { target, err := sttypes.ProtoIDToProtoSpec(sttypes.ProtoID(targetID)) @@ -173,9 +187,9 @@ func (p *Protocol) advertiseLoop() { for { sleep := p.advertise() select { - case <-time.After(sleep): case <-p.closeC: return + case <-time.After(sleep): } } } @@ -209,6 +223,11 @@ func (p *Protocol) supportedProtoIDs() []sttypes.ProtoID { pids := make([]sttypes.ProtoID, 0, len(vs)) for _, v := range vs { pids = append(pids, p.protoIDByVersion(v)) + // beacon node needs to inform shard nodes about it supports them as well for EpochChain + // basically beacon node can accept connection from shard nodes to share last epoch blocks + if p.beaconNode { + pids = append(pids, p.protoIDByVersionForShardNodes(v)) + } } return pids } @@ -228,6 +247,17 @@ func (p *Protocol) protoIDByVersion(v *version.Version) sttypes.ProtoID { return spec.ToProtoID() } +func (p *Protocol) protoIDByVersionForShardNodes(v *version.Version) sttypes.ProtoID { + spec := sttypes.ProtoSpec{ + Service: serviceSpecifier, + NetworkType: p.config.Network, + ShardID: p.config.ShardID, + Version: v, + BeaconNode: false, + } + return spec.ToProtoID() +} + // RemoveStream removes the stream of the given stream ID // TODO: add reason to parameters func (p *Protocol) RemoveStream(stID sttypes.StreamID) { diff --git a/p2p/stream/types/interface.go b/p2p/stream/types/interface.go index 424382cc8c..d7b60f78c3 100644 --- a/p2p/stream/types/interface.go +++ b/p2p/stream/types/interface.go @@ -13,6 +13,8 @@ type Protocol interface { Specifier() string Version() *version.Version ProtoID() ProtoID + // ShardProtoID() ProtoID + IsBeaconNode() bool Match(string) bool HandleStream(st libp2p_network.Stream) } From c9c57f81385ecbfdae7220402c9f9e50d0f483e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Thu, 12 Jan 2023 08:41:46 +0800 Subject: [PATCH 185/420] fix p2p discovery test issue --- p2p/discovery/discovery_test.go | 10 +++++++++- p2p/discovery/option_test.go | 6 +++--- p2p/stream/common/streammanager/streammanager.go | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/p2p/discovery/discovery_test.go b/p2p/discovery/discovery_test.go index 6ea902375f..f2c2f5b99e 100644 --- a/p2p/discovery/discovery_test.go +++ b/p2p/discovery/discovery_test.go @@ -3,9 +3,11 @@ package discovery // TODO: test this module import ( + "context" "testing" "github.com/libp2p/go-libp2p" + dht "github.com/libp2p/go-libp2p-kad-dht" ) func TestNewDHTDiscovery(t *testing.T) { @@ -13,7 +15,13 @@ func TestNewDHTDiscovery(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = NewDHTDiscovery(host, DHTConfig{}) + ctx, cancel := context.WithCancel(context.Background()) + var idht *dht.IpfsDHT + idht, err = dht.New(ctx, host) + if err != nil { + t.Fatal(err) + } + _, err = NewDHTDiscovery(ctx, cancel, host, idht, DHTConfig{}) if err != nil { t.Fatal(err) } diff --git a/p2p/discovery/option_test.go b/p2p/discovery/option_test.go index 747d7ca957..e782151f59 100644 --- a/p2p/discovery/option_test.go +++ b/p2p/discovery/option_test.go @@ -40,14 +40,14 @@ func TestDHTOption_getLibp2pRawOptions(t *testing.T) { opt: DHTConfig{ BootNodes: testAddrStr, }, - expLen: 1, + expLen: 2, }, { opt: DHTConfig{ BootNodes: testAddrStr, DataStoreFile: &validPath, }, - expLen: 2, + expLen: 3, }, { opt: DHTConfig{ @@ -58,7 +58,7 @@ func TestDHTOption_getLibp2pRawOptions(t *testing.T) { }, } for i, test := range tests { - opts, err := test.opt.getLibp2pRawOptions() + opts, err := test.opt.GetLibp2pRawOptions() if assErr := assertError(err, test.expErr); assErr != nil { t.Errorf("Test %v: %v", i, assErr) } diff --git a/p2p/stream/common/streammanager/streammanager.go b/p2p/stream/common/streammanager/streammanager.go index 064cb0c2d0..98e20cbaf9 100644 --- a/p2p/stream/common/streammanager/streammanager.go +++ b/p2p/stream/common/streammanager/streammanager.go @@ -323,7 +323,7 @@ func (sm *streamManager) discoverAndSetupStream(discCtx context.Context) (int, e // The ctx here is using the module context instead of discover context err := sm.setupStreamWithPeer(sm.ctx, pid) if err != nil { - sm.coolDownCache.Add(peer.ID) + sm.coolDownCache.Add(pid) sm.logger.Warn().Err(err).Str("peerID", string(pid)).Msg("failed to setup stream with peer") return } From 7950394dc62382cf0b779ab9098605fa9c004ef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Fri, 13 Jan 2023 16:17:07 +0800 Subject: [PATCH 186/420] add MaxAdvertiseWaitTime to handle advertisements interval and address stream connection issue --- api/service/stagedstreamsync/const.go | 10 +- api/service/stagedstreamsync/downloader.go | 22 ++--- cmd/harmony/config.go | 2 + cmd/harmony/config_migrations.go | 8 ++ cmd/harmony/default.go | 109 ++++++++++++--------- cmd/harmony/flags.go | 9 ++ cmd/harmony/main.go | 21 ++-- internal/configs/harmony/harmony.go | 23 ++--- p2p/stream/protocols/sync/protocol.go | 17 ++-- rosetta/infra/harmony-mainnet.conf | 3 +- rosetta/infra/harmony-pstn.conf | 3 +- 11 files changed, 137 insertions(+), 90 deletions(-) diff --git a/api/service/stagedstreamsync/const.go b/api/service/stagedstreamsync/const.go index ed68d80b13..0e6bc6e2cf 100644 --- a/api/service/stagedstreamsync/const.go +++ b/api/service/stagedstreamsync/const.go @@ -36,11 +36,11 @@ type ( ServerOnly bool // parameters - Network nodeconfig.NetworkType - Concurrency int // Number of concurrent sync requests - MinStreams int // Minimum number of streams to do sync - InitStreams int // Number of streams requirement for initial bootstrap - + Network nodeconfig.NetworkType + Concurrency int // Number of concurrent sync requests + MinStreams int // Minimum number of streams to do sync + InitStreams int // Number of streams requirement for initial bootstrap + MaxAdvertiseWaitTime int // maximum time duration between protocol advertisements // stream manager config SmSoftLowCap int SmHardLowCap int diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go index f53a5c41a8..cfcd180c0f 100644 --- a/api/service/stagedstreamsync/downloader.go +++ b/api/service/stagedstreamsync/downloader.go @@ -41,17 +41,17 @@ func NewDownloader(host p2p.Host, bc core.BlockChain, isBeaconNode bool, config config.fixValues() sp := sync.NewProtocol(sync.Config{ - Chain: bc, - Host: host.GetP2PHost(), - Discovery: host.GetDiscovery(), - ShardID: nodeconfig.ShardID(bc.ShardID()), - Network: config.Network, - BeaconNode: isBeaconNode, - - SmSoftLowCap: config.SmSoftLowCap, - SmHardLowCap: config.SmHardLowCap, - SmHiCap: config.SmHiCap, - DiscBatch: config.SmDiscBatch, + Chain: bc, + Host: host.GetP2PHost(), + Discovery: host.GetDiscovery(), + ShardID: nodeconfig.ShardID(bc.ShardID()), + Network: config.Network, + BeaconNode: isBeaconNode, + MaxAdvertiseWaitTime: config.MaxAdvertiseWaitTime, + SmSoftLowCap: config.SmSoftLowCap, + SmHardLowCap: config.SmHardLowCap, + SmHiCap: config.SmHiCap, + DiscBatch: config.SmDiscBatch, }) host.AddStreamProtocol(sp) diff --git a/cmd/harmony/config.go b/cmd/harmony/config.go index dea5fb2e39..30c72c2e25 100644 --- a/cmd/harmony/config.go +++ b/cmd/harmony/config.go @@ -139,6 +139,8 @@ func getDefaultSyncConfig(nt nodeconfig.NetworkType) harmonyconfig.SyncConfig { return defaultTestNetSyncConfig case nodeconfig.Localnet: return defaultLocalNetSyncConfig + case nodeconfig.Partner: + return defaultPartnerSyncConfig default: return defaultElseSyncConfig } diff --git a/cmd/harmony/config_migrations.go b/cmd/harmony/config_migrations.go index eb98ccb794..3a256f49d2 100644 --- a/cmd/harmony/config_migrations.go +++ b/cmd/harmony/config_migrations.go @@ -318,6 +318,14 @@ func init() { return confTree } + migrations["2.5.10"] = func(confTree *toml.Tree) *toml.Tree { + if confTree.Get("Sync.MaxAdvertiseWaitTime") == nil { + confTree.Set("Sync.MaxAdvertiseWaitTime", defaultConfig.Sync.MaxAdvertiseWaitTime) + } + confTree.Set("Version", "2.5.11") + return confTree + } + // check that the latest version here is the same as in default.go largestKey := getNextVersion(migrations) if largestKey != tomlConfigVersion { diff --git a/cmd/harmony/default.go b/cmd/harmony/default.go index 7de12af9c2..7bdcc02d0d 100644 --- a/cmd/harmony/default.go +++ b/cmd/harmony/default.go @@ -5,7 +5,7 @@ import ( nodeconfig "github.com/harmony-one/harmony/internal/configs/node" ) -const tomlConfigVersion = "2.5.10" +const tomlConfigVersion = "2.5.11" const ( defNetworkType = nodeconfig.Mainnet @@ -159,59 +159,78 @@ var defaultStagedSyncConfig = harmonyconfig.StagedSyncConfig{ var ( defaultMainnetSyncConfig = harmonyconfig.SyncConfig{ - Enabled: false, - Downloader: false, - StagedSync: false, - StagedSyncCfg: defaultStagedSyncConfig, - Concurrency: 6, - MinPeers: 6, - InitStreams: 8, - DiscSoftLowCap: 8, - DiscHardLowCap: 6, - DiscHighCap: 128, - DiscBatch: 8, + Enabled: false, + Downloader: false, + StagedSync: false, + StagedSyncCfg: defaultStagedSyncConfig, + Concurrency: 6, + MinPeers: 6, + InitStreams: 8, + MaxAdvertiseWaitTime: 60, //minutes + DiscSoftLowCap: 8, + DiscHardLowCap: 6, + DiscHighCap: 128, + DiscBatch: 8, } defaultTestNetSyncConfig = harmonyconfig.SyncConfig{ - Enabled: true, - Downloader: false, - StagedSync: false, - StagedSyncCfg: defaultStagedSyncConfig, - Concurrency: 2, - MinPeers: 2, - InitStreams: 2, - DiscSoftLowCap: 2, - DiscHardLowCap: 2, - DiscHighCap: 1024, - DiscBatch: 3, + Enabled: true, + Downloader: false, + StagedSync: false, + StagedSyncCfg: defaultStagedSyncConfig, + Concurrency: 2, + MinPeers: 2, + InitStreams: 2, + MaxAdvertiseWaitTime: 5, //minutes + DiscSoftLowCap: 2, + DiscHardLowCap: 2, + DiscHighCap: 1024, + DiscBatch: 3, } defaultLocalNetSyncConfig = harmonyconfig.SyncConfig{ - Enabled: true, - Downloader: true, - StagedSync: false, - StagedSyncCfg: defaultStagedSyncConfig, - Concurrency: 4, - MinPeers: 5, - InitStreams: 5, - DiscSoftLowCap: 5, - DiscHardLowCap: 5, - DiscHighCap: 1024, - DiscBatch: 8, + Enabled: true, + Downloader: true, + StagedSync: false, + StagedSyncCfg: defaultStagedSyncConfig, + Concurrency: 4, + MinPeers: 5, + InitStreams: 5, + MaxAdvertiseWaitTime: 5, //minutes + DiscSoftLowCap: 5, + DiscHardLowCap: 5, + DiscHighCap: 1024, + DiscBatch: 8, + } + + defaultPartnerSyncConfig = harmonyconfig.SyncConfig{ + Enabled: true, + Downloader: true, + StagedSync: false, + StagedSyncCfg: defaultStagedSyncConfig, + Concurrency: 4, + MinPeers: 2, + InitStreams: 2, + MaxAdvertiseWaitTime: 2, //minutes + DiscSoftLowCap: 2, + DiscHardLowCap: 2, + DiscHighCap: 1024, + DiscBatch: 4, } defaultElseSyncConfig = harmonyconfig.SyncConfig{ - Enabled: true, - Downloader: true, - StagedSync: false, - StagedSyncCfg: defaultStagedSyncConfig, - Concurrency: 4, - MinPeers: 4, - InitStreams: 4, - DiscSoftLowCap: 4, - DiscHardLowCap: 4, - DiscHighCap: 1024, - DiscBatch: 8, + Enabled: true, + Downloader: true, + StagedSync: false, + StagedSyncCfg: defaultStagedSyncConfig, + Concurrency: 4, + MinPeers: 4, + InitStreams: 4, + MaxAdvertiseWaitTime: 2, //minutes + DiscSoftLowCap: 4, + DiscHardLowCap: 4, + DiscHighCap: 1024, + DiscBatch: 8, } ) diff --git a/cmd/harmony/flags.go b/cmd/harmony/flags.go index 26a250d951..168081660c 100644 --- a/cmd/harmony/flags.go +++ b/cmd/harmony/flags.go @@ -1692,6 +1692,11 @@ var ( Usage: "Initial shard-wise number of peers to start syncing", Hidden: true, } + syncMaxAdvertiseWaitTimeFlag = cli.IntFlag{ + Name: "sync.max-advertise-wait-time", + Usage: "The max time duration between two advertises for each p2p peer to tell other nodes what protocols it supports", + Hidden: true, + } syncDiscSoftLowFlag = cli.IntFlag{ Name: "sync.disc.soft-low-cap", Usage: "Soft low cap for sync stream management", @@ -1740,6 +1745,10 @@ func applySyncFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { config.Sync.InitStreams = cli.GetIntFlagValue(cmd, syncInitStreamsFlag) } + if cli.IsFlagChanged(cmd, syncMaxAdvertiseWaitTimeFlag) { + config.Sync.MaxAdvertiseWaitTime = cli.GetIntFlagValue(cmd, syncMaxAdvertiseWaitTimeFlag) + } + if cli.IsFlagChanged(cmd, syncDiscSoftLowFlag) { config.Sync.DiscSoftLowCap = cli.GetIntFlagValue(cmd, syncDiscSoftLowFlag) } diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 6ab3ed1024..dbb18227d2 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -918,16 +918,17 @@ func setupStagedSyncService(node *node.Node, host p2p.Host, hc harmonyconfig.Har } sConfig := stagedstreamsync.Config{ - ServerOnly: !hc.Sync.Downloader, - Network: nodeconfig.NetworkType(hc.Network.NetworkType), - Concurrency: hc.Sync.Concurrency, - MinStreams: hc.Sync.MinPeers, - InitStreams: hc.Sync.InitStreams, - SmSoftLowCap: hc.Sync.DiscSoftLowCap, - SmHardLowCap: hc.Sync.DiscHardLowCap, - SmHiCap: hc.Sync.DiscHighCap, - SmDiscBatch: hc.Sync.DiscBatch, - LogProgress: node.NodeConfig.LogProgress, + ServerOnly: !hc.Sync.Downloader, + Network: nodeconfig.NetworkType(hc.Network.NetworkType), + Concurrency: hc.Sync.Concurrency, + MinStreams: hc.Sync.MinPeers, + InitStreams: hc.Sync.InitStreams, + MaxAdvertiseWaitTime: hc.Sync.MaxAdvertiseWaitTime, + SmSoftLowCap: hc.Sync.DiscSoftLowCap, + SmHardLowCap: hc.Sync.DiscHardLowCap, + SmHiCap: hc.Sync.DiscHighCap, + SmDiscBatch: hc.Sync.DiscBatch, + LogProgress: node.NodeConfig.LogProgress, } // If we are running side chain, we will need to do some extra works for beacon diff --git a/internal/configs/harmony/harmony.go b/internal/configs/harmony/harmony.go index 90569b96f4..73ddd442dd 100644 --- a/internal/configs/harmony/harmony.go +++ b/internal/configs/harmony/harmony.go @@ -222,17 +222,18 @@ type PrometheusConfig struct { type SyncConfig struct { // TODO: Remove this bool after stream sync is fully up. - Enabled bool // enable the stream sync protocol - Downloader bool // start the sync downloader client - StagedSync bool // use staged sync - StagedSyncCfg StagedSyncConfig // staged sync configurations - Concurrency int // concurrency used for stream sync protocol - MinPeers int // minimum streams to start a sync task. - InitStreams int // minimum streams in bootstrap to start sync loop. - DiscSoftLowCap int // when number of streams is below this value, spin discover during check - DiscHardLowCap int // when removing stream, num is below this value, spin discovery immediately - DiscHighCap int // upper limit of streams in one sync protocol - DiscBatch int // size of each discovery + Enabled bool // enable the stream sync protocol + Downloader bool // start the sync downloader client + StagedSync bool // use staged sync + StagedSyncCfg StagedSyncConfig // staged sync configurations + Concurrency int // concurrency used for stream sync protocol + MinPeers int // minimum streams to start a sync task. + InitStreams int // minimum streams in bootstrap to start sync loop. + MaxAdvertiseWaitTime int // maximum time duration between advertisements + DiscSoftLowCap int // when number of streams is below this value, spin discover during check + DiscHardLowCap int // when removing stream, num is below this value, spin discovery immediately + DiscHighCap int // upper limit of streams in one sync protocol + DiscBatch int // size of each discovery } type StagedSyncConfig struct { diff --git a/p2p/stream/protocols/sync/protocol.go b/p2p/stream/protocols/sync/protocol.go index 2a984a5818..80ea0927bc 100644 --- a/p2p/stream/protocols/sync/protocol.go +++ b/p2p/stream/protocols/sync/protocol.go @@ -59,12 +59,13 @@ type ( // Config is the sync protocol config Config struct { - Chain engine.ChainReader - Host libp2p_host.Host - Discovery discovery.Discovery - ShardID nodeconfig.ShardID - Network nodeconfig.NetworkType - BeaconNode bool + Chain engine.ChainReader + Host libp2p_host.Host + Discovery discovery.Discovery + ShardID nodeconfig.ShardID + Network nodeconfig.NetworkType + BeaconNode bool + MaxAdvertiseWaitTime int // stream manager config SmSoftLowCap int SmHardLowCap int @@ -186,6 +187,10 @@ func (p *Protocol) HandleStream(raw libp2p_network.Stream) { func (p *Protocol) advertiseLoop() { for { sleep := p.advertise() + maxSleepTime := time.Duration(p.config.MaxAdvertiseWaitTime) * time.Minute + if sleep > maxSleepTime { + sleep = maxSleepTime + } select { case <-p.closeC: return diff --git a/rosetta/infra/harmony-mainnet.conf b/rosetta/infra/harmony-mainnet.conf index ddb2f87eea..3e7085ba63 100644 --- a/rosetta/infra/harmony-mainnet.conf +++ b/rosetta/infra/harmony-mainnet.conf @@ -1,4 +1,4 @@ -Version = "2.5.10" +Version = "2.5.11" [BLSKeys] KMSConfigFile = "" @@ -105,6 +105,7 @@ Version = "2.5.10" Enabled = false InitStreams = 8 MinPeers = 5 + MaxAdvertiseWaitTime = 30 [TxPool] AccountSlots = 16 diff --git a/rosetta/infra/harmony-pstn.conf b/rosetta/infra/harmony-pstn.conf index 1c029f8281..458e826354 100644 --- a/rosetta/infra/harmony-pstn.conf +++ b/rosetta/infra/harmony-pstn.conf @@ -1,4 +1,4 @@ -Version = "2.5.10" +Version = "2.5.11" [BLSKeys] KMSConfigFile = "" @@ -105,6 +105,7 @@ Version = "2.5.10" Enabled = false InitStreams = 8 MinPeers = 2 + MaxAdvertiseWaitTime = 30 [TxPool] AccountSlots = 16 From da6640557109516fb3dcaed822064cf2a9a20406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Mon, 16 Jan 2023 23:18:02 +0800 Subject: [PATCH 187/420] terminal print the peer id and proto id --- p2p/stream/common/streammanager/streammanager.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/p2p/stream/common/streammanager/streammanager.go b/p2p/stream/common/streammanager/streammanager.go index 98e20cbaf9..e57919a76f 100644 --- a/p2p/stream/common/streammanager/streammanager.go +++ b/p2p/stream/common/streammanager/streammanager.go @@ -73,8 +73,12 @@ func newStreamManager(pid sttypes.ProtoID, host host, pf peerFinder, handleStrea Str("protocol ID", string(pid)).Logger() protoSpec, _ := sttypes.ProtoIDToProtoSpec(pid) - fmt.Println("my peer id: ", host.ID().String()) - fmt.Println("my proto id: ", pid) + + // if it is a beacon node or shard node, print the peer id and proto id + if protoSpec.BeaconNode || protoSpec.ShardID != shard.BeaconChainShardID { + fmt.Println("My peer id: ", host.ID().String()) + fmt.Println("My proto id: ", pid) + } return &streamManager{ myProtoID: pid, From 0c9fb193bb2f0933b9820cc6c4f45da656f17edd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Thu, 19 Jan 2023 23:42:29 +0800 Subject: [PATCH 188/420] fix boot complete message when node is shut down --- api/service/stagedstreamsync/downloader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go index cfcd180c0f..a20b4ac79a 100644 --- a/api/service/stagedstreamsync/downloader.go +++ b/api/service/stagedstreamsync/downloader.go @@ -95,7 +95,6 @@ func NewDownloader(host p2p.Host, bc core.BlockChain, isBeaconNode bool, config func (d *Downloader) Start() { go func() { d.waitForBootFinish() - fmt.Printf("boot completed for shard %d, %d streams are connected\n", d.bc.ShardID(), d.syncProtocol.NumStreams()) d.loop() }() @@ -178,6 +177,7 @@ func (d *Downloader) waitForBootFinish() { case <-checkCh: if d.syncProtocol.NumStreams() >= d.config.InitStreams { + fmt.Printf("boot completed for shard %d ( %d streams are connected )\n", d.bc.ShardID(), d.syncProtocol.NumStreams()) return } case <-d.closeC: From 5625f6ed9449362aa2a546da2d5752b58dcc48fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Fri, 20 Jan 2023 10:50:03 +0800 Subject: [PATCH 189/420] add new config option ( ForceReachabilityPublic ) to fix local-net consensus issue --- cmd/bootnode/main.go | 13 ++++++---- cmd/harmony/default.go | 10 ++++---- cmd/harmony/main.go | 8 ++++++ p2p/host.go | 57 ++++++++++++++++++++++++++++++------------ test/deploy.sh | 2 +- 5 files changed, 63 insertions(+), 27 deletions(-) diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index 6e3a8bbbc5..92d6674b94 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -102,6 +102,7 @@ func main() { verbosity := flag.Int("verbosity", 5, "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail (default: 5)") logConn := flag.Bool("log_conn", false, "log incoming/outgoing connections") maxConnPerIP := flag.Int("max_conn_per_ip", 10, "max connections number for same ip") + forceReachabilityPublic := flag.Bool("force_public", false, "forcing the local node to believe it is reachable externally") flag.Parse() @@ -124,12 +125,14 @@ func main() { // For bootstrap nodes, we shall keep .dht file. dataStorePath := fmt.Sprintf(".dht-%s-%s", *ip, *port) selfPeer := p2p.Peer{IP: *ip, Port: *port} + host, err := p2p.NewHost(p2p.HostConfig{ - Self: &selfPeer, - BLSKey: privKey, - BootNodes: nil, // Boot nodes have no boot nodes :) Will be connected when other nodes joined - DataStoreFile: &dataStorePath, - MaxConnPerIP: *maxConnPerIP, + Self: &selfPeer, + BLSKey: privKey, + BootNodes: nil, // Boot nodes have no boot nodes :) Will be connected when other nodes joined + DataStoreFile: &dataStorePath, + MaxConnPerIP: *maxConnPerIP, + ForceReachabilityPublic: *forceReachabilityPublic, }) if err != nil { utils.FatalErrMsg(err, "cannot initialize network") diff --git a/cmd/harmony/default.go b/cmd/harmony/default.go index 7bdcc02d0d..2a6c7a75c3 100644 --- a/cmd/harmony/default.go +++ b/cmd/harmony/default.go @@ -191,14 +191,14 @@ var ( defaultLocalNetSyncConfig = harmonyconfig.SyncConfig{ Enabled: true, Downloader: true, - StagedSync: false, + StagedSync: true, StagedSyncCfg: defaultStagedSyncConfig, Concurrency: 4, - MinPeers: 5, - InitStreams: 5, + MinPeers: 4, + InitStreams: 4, MaxAdvertiseWaitTime: 5, //minutes - DiscSoftLowCap: 5, - DiscHardLowCap: 5, + DiscSoftLowCap: 4, + DiscHardLowCap: 4, DiscHighCap: 1024, DiscBatch: 8, } diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index dbb18227d2..8543158b3a 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -629,6 +629,13 @@ func createGlobalConfig(hc harmonyconfig.HarmonyConfig) (*nodeconfig.ConfigType, Port: strconv.Itoa(hc.P2P.Port), ConsensusPubKey: nodeConfig.ConsensusPriKey[0].Pub.Object, } + + // for local-net the node has to be forced to assume it is public reachable + forceReachabilityPublic := false + if hc.Network.NetworkType == nodeconfig.Localnet { + forceReachabilityPublic = true + } + myHost, err = p2p.NewHost(p2p.HostConfig{ Self: &selfPeer, BLSKey: nodeConfig.P2PPriKey, @@ -639,6 +646,7 @@ func createGlobalConfig(hc harmonyconfig.HarmonyConfig) (*nodeconfig.ConfigType, DisablePrivateIPScan: hc.P2P.DisablePrivateIPScan, MaxPeers: hc.P2P.MaxPeers, WaitForEachPeerToConnect: hc.P2P.WaitForEachPeerToConnect, + ForceReachabilityPublic: forceReachabilityPublic, }) if err != nil { return nil, errors.Wrap(err, "cannot create P2P network host") diff --git a/p2p/host.go b/p2p/host.go index fcd2ea3b4d..bab1da691c 100644 --- a/p2p/host.go +++ b/p2p/host.go @@ -16,6 +16,7 @@ import ( "github.com/libp2p/go-libp2p-core/routing" dht "github.com/libp2p/go-libp2p-kad-dht" libp2p_pubsub "github.com/libp2p/go-libp2p-pubsub" + libp2p_config "github.com/libp2p/go-libp2p/config" libp2p_crypto "github.com/libp2p/go-libp2p/core/crypto" libp2p_host "github.com/libp2p/go-libp2p/core/host" libp2p_network "github.com/libp2p/go-libp2p/core/network" @@ -96,6 +97,7 @@ type HostConfig struct { DisablePrivateIPScan bool MaxPeers int64 WaitForEachPeerToConnect bool + ForceReachabilityPublic bool } func init() { @@ -112,6 +114,19 @@ func init() { libp2p_pubsub.GossipSubMaxIHaveLength = 1000 } +func forceReachabilityPublic(f bool) libp2p_config.Option { + if f { + return func(cfg *libp2p_config.Config) error { + public := libp2p_network.Reachability(libp2p_network.ReachabilityPublic) + cfg.AutoNATConfig.ForceReachability = &public + return nil + } + } + return func(p2pConfig *libp2p_config.Config) error { + return nil + } +} + // NewHost .. func NewHost(cfg HostConfig) (Host, error) { var ( @@ -128,8 +143,8 @@ func NewHost(cfg HostConfig) (Host, error) { ctx, cancel := context.WithCancel(context.Background()) // TODO: move low and high to configs connmgr, err := connmgr.NewConnManager( - int(10), // LowWater - int(10000)*cfg.MaxConnPerIP, // HighWater, + int(cfg.MaxConnPerIP), // LowWater + int(1024)*cfg.MaxConnPerIP, // HighWater, connmgr.WithGracePeriod(time.Minute), ) if err != nil { @@ -138,16 +153,16 @@ func NewHost(cfg HostConfig) (Host, error) { } var idht *dht.IpfsDHT var opt discovery.DHTConfig - p2pHost, err := libp2p.New( + p2pHostConfig := []libp2p.Option{ listenAddr, libp2p.Identity(key), - // support TLS connections + // Support TLS connections libp2p.Security(libp2ptls.ID, libp2ptls.New), - // support noise connections + // Support noise connections libp2p.Security(noise.ID, noise.New), - // support any other default transports (TCP) + // Support any other default transports (TCP) libp2p.DefaultTransports, - // Let's prevent our peer from having too many + // Prevent the peer from having too many // connections by attaching a connection manager. libp2p.ConnectionManager(connmgr), // Attempt to open ports using uPNP for NATed hosts. @@ -165,8 +180,7 @@ func NewHost(cfg HostConfig) (Host, error) { idht, err = dht.New(ctx, h, opts...) return idht, err }), - - // to help other peers to figure out if they are behind + // To help other peers to figure out if they are behind // NATs, launch the server-side of AutoNAT too (AutoRelay // already runs the client) // This service is highly rate-limited and should not cause any @@ -174,14 +188,23 @@ func NewHost(cfg HostConfig) (Host, error) { libp2p.EnableNATService(), // Bandwidth Reporter libp2p.BandwidthReporter(newCounter()), - // ForceReachabilityPublic overrides automatic reachability detection in the AutoNAT subsystem, - // forcing the local node to believe it is reachable externally. - // libp2p.ForceReachabilityPublic(), - // libp2p.DisableRelay(), + // Enable relay service, to disable relay we can use libp2p.DisableRelay() libp2p.EnableRelayService(), - // prevent dialing of public addresses - // libp2p.ConnectionGater(NewGater(cfg.DisablePrivateIPScan)), - ) + } + + if cfg.ForceReachabilityPublic { + // ForceReachabilityPublic overrides automatic reachability detection in the AutoNAT subsystem, + // forcing the local node to believe it is reachable externally + p2pHostConfig = append(p2pHostConfig, libp2p.ForceReachabilityPublic()) + } + + if cfg.DisablePrivateIPScan { + // Prevent dialing of public addresses + p2pHostConfig = append(p2pHostConfig, libp2p.ConnectionGater(NewGater(cfg.DisablePrivateIPScan))) + } + + // create p2p host + p2pHost, err := libp2p.New(p2pHostConfig...) if err != nil { cancel() return nil, errors.Wrapf(err, "cannot initialize libp2p host") @@ -190,6 +213,7 @@ func NewHost(cfg HostConfig) (Host, error) { disc, err := discovery.NewDHTDiscovery(ctx, cancel, p2pHost, idht, opt) if err != nil { cancel() + p2pHost.Close() return nil, errors.Wrap(err, "cannot create DHT discovery") } @@ -230,6 +254,7 @@ func NewHost(cfg HostConfig) (Host, error) { pubsub, err := libp2p_pubsub.NewGossipSub(ctx, p2pHost, options...) if err != nil { cancel() + p2pHost.Close() return nil, errors.Wrapf(err, "cannot initialize libp2p pub-sub") } diff --git a/test/deploy.sh b/test/deploy.sh index 6bbb12eb9b..9ab6943110 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -54,7 +54,7 @@ function setup() { function launch_bootnode() { echo "launching boot node ..." - ${DRYRUN} ${ROOT}/bin/bootnode -port 19876 -max_conn_per_ip 100 >"${log_folder}"/bootnode.log 2>&1 | tee -a "${LOG_FILE}" & + ${DRYRUN} ${ROOT}/bin/bootnode -port 19876 -max_conn_per_ip 100 -force_public true >"${log_folder}"/bootnode.log 2>&1 | tee -a "${LOG_FILE}" & sleep 1 BN_MA=$(grep "BN_MA" "${log_folder}"/bootnode.log | awk -F\= ' { print $2 } ') echo "bootnode launched." + " $BN_MA" From 9084066a50575397d6ef0a3ead861b873373c891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Thu, 26 Jan 2023 11:17:45 +0800 Subject: [PATCH 190/420] fix self query issue --- api/service/legacysync/epoch_syncing.go | 6 +++-- api/service/legacysync/helpers.go | 6 +++-- api/service/legacysync/syncing.go | 24 ++++++++++++-------- api/service/legacysync/syncing_test.go | 4 +++- api/service/stagedsync/stagedsync.go | 10 +++++++-- api/service/stagedsync/sync_config.go | 7 ++++++ cmd/harmony/main.go | 3 ++- node/node.go | 2 +- node/node_syncing.go | 29 +++++++++++++++++++++---- 9 files changed, 69 insertions(+), 22 deletions(-) diff --git a/api/service/legacysync/epoch_syncing.go b/api/service/legacysync/epoch_syncing.go index 8bf7ae1458..641fb58890 100644 --- a/api/service/legacysync/epoch_syncing.go +++ b/api/service/legacysync/epoch_syncing.go @@ -14,6 +14,8 @@ import ( "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/shard" "github.com/pkg/errors" + + libp2p_peer "github.com/libp2p/go-libp2p/core/peer" ) type EpochSync struct { @@ -202,8 +204,8 @@ func processWithPayload(payload [][]byte, bc core.BlockChain) error { } // CreateSyncConfig creates SyncConfig for StateSync object. -func (ss *EpochSync) CreateSyncConfig(peers []p2p.Peer, shardID uint32, waitForEachPeerToConnect bool) error { +func (ss *EpochSync) CreateSyncConfig(peers []p2p.Peer, shardID uint32, selfPeerID libp2p_peer.ID, waitForEachPeerToConnect bool) error { var err error - ss.syncConfig, err = createSyncConfig(ss.syncConfig, peers, shardID, waitForEachPeerToConnect) + ss.syncConfig, err = createSyncConfig(ss.syncConfig, peers, shardID, selfPeerID, waitForEachPeerToConnect) return err } diff --git a/api/service/legacysync/helpers.go b/api/service/legacysync/helpers.go index ca66e0ddcb..4699257f8b 100644 --- a/api/service/legacysync/helpers.go +++ b/api/service/legacysync/helpers.go @@ -9,6 +9,7 @@ import ( "github.com/harmony-one/harmony/api/service/legacysync/downloader" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" + libp2p_peer "github.com/libp2p/go-libp2p/core/peer" "github.com/pkg/errors" ) @@ -51,7 +52,7 @@ func getMaxPeerHeight(syncConfig *SyncConfig) uint64 { return maxHeight } -func createSyncConfig(syncConfig *SyncConfig, peers []p2p.Peer, shardID uint32, waitForEachPeerToConnect bool) (*SyncConfig, error) { +func createSyncConfig(syncConfig *SyncConfig, peers []p2p.Peer, shardID uint32, selfPeerID libp2p_peer.ID, waitForEachPeerToConnect bool) (*SyncConfig, error) { // sanity check to ensure no duplicate peers if err := checkPeersDuplicity(peers); err != nil { return syncConfig, err @@ -61,6 +62,7 @@ func createSyncConfig(syncConfig *SyncConfig, peers []p2p.Peer, shardID uint32, targetSize, peers := limitNumPeers(peers, randSeed) utils.Logger().Debug(). + Str("self peer ID", string(selfPeerID)). Int("peers count", len(peers)). Int("target size", targetSize). Uint32("shardID", shardID). @@ -72,7 +74,7 @@ func createSyncConfig(syncConfig *SyncConfig, peers []p2p.Peer, shardID uint32, if syncConfig != nil { syncConfig.CloseConnections() } - syncConfig = NewSyncConfig(shardID, nil) + syncConfig = NewSyncConfig(shardID, selfPeerID, nil) if !waitForEachPeerToConnect { var wg sync.WaitGroup diff --git a/api/service/legacysync/syncing.go b/api/service/legacysync/syncing.go index 8afaa3a95b..0f004bfb4f 100644 --- a/api/service/legacysync/syncing.go +++ b/api/service/legacysync/syncing.go @@ -25,6 +25,7 @@ import ( "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/node/worker" "github.com/harmony-one/harmony/p2p" + libp2p_peer "github.com/libp2p/go-libp2p/core/peer" "github.com/pkg/errors" ) @@ -110,14 +111,16 @@ type SyncConfig struct { // SyncPeerConfig itself is guarded by its own mutex. mtx sync.RWMutex - peers []*SyncPeerConfig - shardID uint32 + peers []*SyncPeerConfig + shardID uint32 + selfPeerID libp2p_peer.ID } -func NewSyncConfig(shardID uint32, peers []*SyncPeerConfig) *SyncConfig { +func NewSyncConfig(shardID uint32, selfPeerID libp2p_peer.ID, peers []*SyncPeerConfig) *SyncConfig { return &SyncConfig{ - peers: peers, - shardID: shardID, + peers: peers, + shardID: shardID, + selfPeerID: selfPeerID, } } @@ -135,6 +138,9 @@ func (sc *SyncConfig) AddPeer(peer *SyncPeerConfig) { if peer.IsEqual(p2) { return } + if peer.peer.PeerID == sc.selfPeerID { + return + } } sc.peers = append(sc.peers, peer) } @@ -192,7 +198,7 @@ func (sc *SyncConfig) RemovePeer(peer *SyncPeerConfig, reason string) { } // CreateStateSync returns the implementation of StateSyncInterface interface. -func CreateStateSync(bc blockChain, ip string, port string, peerHash [20]byte, isExplorer bool, role nodeconfig.Role) *StateSync { +func CreateStateSync(bc blockChain, ip string, port string, peerHash [20]byte, peerID libp2p_peer.ID, isExplorer bool, role nodeconfig.Role) *StateSync { stateSync := &StateSync{} stateSync.blockChain = bc stateSync.selfip = ip @@ -201,7 +207,7 @@ func CreateStateSync(bc blockChain, ip string, port string, peerHash [20]byte, i stateSync.commonBlocks = make(map[int]*types.Block) stateSync.lastMileBlocks = []*types.Block{} stateSync.isExplorer = isExplorer - stateSync.syncConfig = NewSyncConfig(bc.ShardID(), nil) + stateSync.syncConfig = NewSyncConfig(bc.ShardID(), peerID, nil) stateSync.syncStatus = newSyncStatus(role) return stateSync @@ -366,9 +372,9 @@ func (peerConfig *SyncPeerConfig) GetBlocks(hashes [][]byte) ([][]byte, error) { } // CreateSyncConfig creates SyncConfig for StateSync object. -func (ss *StateSync) CreateSyncConfig(peers []p2p.Peer, shardID uint32, waitForEachPeerToConnect bool) error { +func (ss *StateSync) CreateSyncConfig(peers []p2p.Peer, shardID uint32, selfPeerID libp2p_peer.ID, waitForEachPeerToConnect bool) error { var err error - ss.syncConfig, err = createSyncConfig(ss.syncConfig, peers, shardID, waitForEachPeerToConnect) + ss.syncConfig, err = createSyncConfig(ss.syncConfig, peers, shardID, selfPeerID, waitForEachPeerToConnect) return err } diff --git a/api/service/legacysync/syncing_test.go b/api/service/legacysync/syncing_test.go index bcab494a18..bc17aeaec6 100644 --- a/api/service/legacysync/syncing_test.go +++ b/api/service/legacysync/syncing_test.go @@ -12,6 +12,7 @@ import ( "time" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + peer "github.com/libp2p/go-libp2p-core/peer" "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/harmony/api/service/legacysync/downloader" @@ -128,7 +129,8 @@ func (mockBlockchain) ShardID() uint32 { } func TestCreateStateSync(t *testing.T) { - stateSync := CreateStateSync(mockBlockchain{}, "127.0.0.1", "8000", [20]byte{}, false, nodeconfig.Validator) + pID, _ := peer.IDFromBytes([]byte{}) + stateSync := CreateStateSync(mockBlockchain{}, "127.0.0.1", "8000", [20]byte{}, pID, false, nodeconfig.Validator) if stateSync == nil { t.Error("Unable to create stateSync") diff --git a/api/service/stagedsync/stagedsync.go b/api/service/stagedsync/stagedsync.go index 88df1a671f..c81f0b5b12 100644 --- a/api/service/stagedsync/stagedsync.go +++ b/api/service/stagedsync/stagedsync.go @@ -26,6 +26,8 @@ import ( "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" "github.com/ledgerwatch/erigon-lib/kv" + + libp2p_peer "github.com/libp2p/go-libp2p/core/peer" ) type StagedSync struct { @@ -663,7 +665,7 @@ func (ss *StagedSync) AddNewBlock(peerHash []byte, block *types.Block) { } // CreateSyncConfig creates SyncConfig for StateSync object. -func (ss *StagedSync) CreateSyncConfig(peers []p2p.Peer, shardID uint32, waitForEachPeerToConnect bool) error { +func (ss *StagedSync) CreateSyncConfig(peers []p2p.Peer, shardID uint32, selfPeerID libp2p_peer.ID, waitForEachPeerToConnect bool) error { // sanity check to ensure no duplicate peers if err := checkPeersDuplicity(peers); err != nil { return err @@ -678,6 +680,7 @@ func (ss *StagedSync) CreateSyncConfig(peers []p2p.Peer, shardID uint32, waitFor } utils.Logger().Debug(). + Str("self peer ID", string(selfPeerID)). Int("peers count", len(peers)). Int("target size", targetSize). Msg("[STAGED_SYNC] CreateSyncConfig: len of peers") @@ -685,7 +688,9 @@ func (ss *StagedSync) CreateSyncConfig(peers []p2p.Peer, shardID uint32, waitFor if ss.syncConfig != nil { ss.syncConfig.CloseConnections() } - ss.syncConfig = &SyncConfig{} + ss.syncConfig = &SyncConfig{ + selfPeerID: selfPeerID, + } var connectedPeers int for _, peer := range peers { @@ -694,6 +699,7 @@ func (ss *StagedSync) CreateSyncConfig(peers []p2p.Peer, shardID uint32, waitFor continue } peerConfig := &SyncPeerConfig{ + peer: peer, ip: peer.IP, port: peer.Port, client: client, diff --git a/api/service/stagedsync/sync_config.go b/api/service/stagedsync/sync_config.go index f42737cc1a..91b3a4d739 100644 --- a/api/service/stagedsync/sync_config.go +++ b/api/service/stagedsync/sync_config.go @@ -14,6 +14,8 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" + + libp2p_peer "github.com/libp2p/go-libp2p/core/peer" ) // Constants for syncing. @@ -40,6 +42,7 @@ const ( // SyncPeerConfig is peer config to sync. type SyncPeerConfig struct { + peer p2p.Peer ip string port string peerHash []byte @@ -156,6 +159,7 @@ type SyncConfig struct { mtx sync.RWMutex reservedPeers []*SyncPeerConfig peers []*SyncPeerConfig + selfPeerID libp2p_peer.ID } // AddPeer adds the given sync peer. @@ -168,6 +172,9 @@ func (sc *SyncConfig) AddPeer(peer *SyncPeerConfig) { if peer.IsEqual(p2) { return } + if peer.peer.PeerID == sc.selfPeerID { + return + } } sc.peers = append(sc.peers, peer) } diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 8543158b3a..b68e757422 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -773,7 +773,8 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi currentNode.SyncingPeerProvider = node.NewLocalSyncingPeerProvider( 6000, uint16(selfPort), epochConfig.NumShards(), uint32(epochConfig.NumNodesPerShard())) } else { - currentNode.SyncingPeerProvider = node.NewDNSSyncingPeerProvider(hc.DNSSync.Zone, strconv.Itoa(hc.DNSSync.Port)) + addrs := myHost.GetP2PHost().Addrs() + currentNode.SyncingPeerProvider = node.NewDNSSyncingPeerProvider(hc.DNSSync.Zone, strconv.Itoa(hc.DNSSync.Port), addrs) } currentNode.NodeConfig.DNSZone = hc.DNSSync.Zone diff --git a/node/node.go b/node/node.go index bc80ef81ae..6c2ebcd5fe 100644 --- a/node/node.go +++ b/node/node.go @@ -87,7 +87,7 @@ type ISync interface { UpdateBlockAndStatus(block *types.Block, bc core.BlockChain, verifyAllSig bool) error AddLastMileBlock(block *types.Block) GetActivePeerNumber() int - CreateSyncConfig(peers []p2p.Peer, shardID uint32, waitForEachPeerToConnect bool) error + CreateSyncConfig(peers []p2p.Peer, shardID uint32, selfPeerID libp2p_peer.ID, waitForEachPeerToConnect bool) error SyncLoop(bc core.BlockChain, worker *worker.Worker, isBeacon bool, consensus *consensus.Consensus, loopMinTime time.Duration) IsSynchronized() bool IsSameBlockchainHeight(bc core.BlockChain) (uint64, bool) diff --git a/node/node_syncing.go b/node/node_syncing.go index 2219be96da..a0a8e6c84c 100644 --- a/node/node_syncing.go +++ b/node/node_syncing.go @@ -9,6 +9,7 @@ import ( "time" "github.com/harmony-one/harmony/internal/tikv" + "github.com/multiformats/go-multiaddr" prom "github.com/harmony-one/harmony/api/service/prometheus" "github.com/prometheus/client_golang/prometheus" @@ -105,7 +106,8 @@ func (node *Node) createStateSync(bc core.BlockChain) *legacysync.StateSync { mutatedPort := strconv.Itoa(mySyncPort + legacysync.SyncingPortDifference) role := node.NodeConfig.Role() return legacysync.CreateStateSync(bc, node.SelfPeer.IP, mutatedPort, - node.GetSyncID(), node.NodeConfig.Role() == nodeconfig.ExplorerNode, role) + node.GetSyncID(), node.host.GetID(), + node.NodeConfig.Role() == nodeconfig.ExplorerNode, role) } func (node *Node) createStagedSync(bc core.BlockChain) *stagedsync.StagedSync { @@ -151,14 +153,16 @@ type SyncingPeerProvider interface { // DNSSyncingPeerProvider uses the given DNS zone to resolve syncing peers. type DNSSyncingPeerProvider struct { + selfAddrs []multiaddr.Multiaddr zone, port string lookupHost func(name string) (addrs []string, err error) } // NewDNSSyncingPeerProvider returns a provider that uses given DNS name and // port number to resolve syncing peers. -func NewDNSSyncingPeerProvider(zone, port string) *DNSSyncingPeerProvider { +func NewDNSSyncingPeerProvider(zone, port string, addrs []multiaddr.Multiaddr) *DNSSyncingPeerProvider { return &DNSSyncingPeerProvider{ + selfAddrs: addrs, zone: zone, port: port, lookupHost: net.LookupHost, @@ -174,11 +178,27 @@ func (p *DNSSyncingPeerProvider) SyncingPeers(shardID uint32) (peers []p2p.Peer, "[SYNC] cannot find peers using DNS name %#v", dns) } for _, addr := range addrs { + // no need to have peer itself on the list of connected peers + if p.getSelfAddrIndex(addr, p.port) >= 0 { + continue + } peers = append(peers, p2p.Peer{IP: addr, Port: p.port}) } return peers, nil } +// getSelfAddrIndex returns address index if it is one of self addresses +func (p *DNSSyncingPeerProvider) getSelfAddrIndex(IP string, Port string) int { + peerAddr4 := fmt.Sprintf("/ip4/%s/tcp/%s", IP, Port) + peerAddr6 := fmt.Sprintf("/ip6/%s/tcp/%s", IP, Port) + for addrIndex, addr := range p.selfAddrs { + if addr.String() == peerAddr4 || addr.String() == peerAddr6 { + return addrIndex + } + } + return -1 +} + // LocalSyncingPeerProvider uses localnet deployment convention to synthesize // syncing peers. type LocalSyncingPeerProvider struct { @@ -253,7 +273,8 @@ func (node *Node) doBeaconSyncing() { continue } - if err := node.epochSync.CreateSyncConfig(peers, shard.BeaconChainShardID, node.HarmonyConfig.P2P.WaitForEachPeerToConnect); err != nil { + if err := node.epochSync.CreateSyncConfig(peers, shard.BeaconChainShardID, node.host.GetID(), + node.HarmonyConfig.P2P.WaitForEachPeerToConnect); err != nil { utils.Logger().Warn().Err(err).Msg("[EPOCHSYNC] cannot create beacon sync config") continue } @@ -296,7 +317,7 @@ func (node *Node) doSync(bc core.BlockChain, worker *worker.Worker, willJoinCons Msg("cannot retrieve syncing peers") return } - if err := syncInstance.CreateSyncConfig(peers, shardID, node.HarmonyConfig.P2P.WaitForEachPeerToConnect); err != nil { + if err := syncInstance.CreateSyncConfig(peers, shardID, node.host.GetID(), node.HarmonyConfig.P2P.WaitForEachPeerToConnect); err != nil { utils.Logger().Warn(). Err(err). Interface("peers", peers). From 6f78ac7a1bea8c60faaa5a53aac2ec7b56b0158d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Thu, 26 Jan 2023 13:53:47 +0800 Subject: [PATCH 191/420] fix test NewDNSSyncingPeerProvider --- node/node_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/node/node_test.go b/node/node_test.go index a4f1af70c3..49ba5d164d 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -16,6 +16,7 @@ import ( "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/shard" + "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/assert" ) @@ -69,7 +70,8 @@ func TestNewNode(t *testing.T) { func TestDNSSyncingPeerProvider(t *testing.T) { t.Run("Happy", func(t *testing.T) { - p := NewDNSSyncingPeerProvider("example.com", "1234") + addrs := make([]multiaddr.Multiaddr, 0) + p := NewDNSSyncingPeerProvider("example.com", "1234", addrs) lookupCount := 0 lookupName := "" p.lookupHost = func(name string) (addrs []string, err error) { @@ -92,7 +94,8 @@ func TestDNSSyncingPeerProvider(t *testing.T) { } }) t.Run("LookupError", func(t *testing.T) { - p := NewDNSSyncingPeerProvider("example.com", "1234") + addrs := make([]multiaddr.Multiaddr, 0) + p := NewDNSSyncingPeerProvider("example.com", "1234", addrs) p.lookupHost = func(_ string) ([]string, error) { return nil, errors.New("omg") } From 4bb360d137a7885799cbda4a4272e7e9d7aef22c Mon Sep 17 00:00:00 2001 From: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Date: Thu, 2 Feb 2023 13:11:40 +0000 Subject: [PATCH 192/420] [testnet] disable leader rotation --- internal/params/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/params/config.go b/internal/params/config.go index 3d40a7c6c9..c11daeafb5 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -109,7 +109,7 @@ var ( ChainIdFixEpoch: big.NewInt(0), CrossShardXferPrecompileEpoch: big.NewInt(2), AllowlistEpoch: big.NewInt(2), - LeaderRotationEpoch: big.NewInt(295), + LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 64, FeeCollectEpoch: EpochTBD, } From 5361044376060d1740d235ead9a79560b3c0f55c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Fri, 10 Feb 2023 22:36:54 +0800 Subject: [PATCH 193/420] fix discovery issue for legacy sync --- api/service/legacysync/epoch_syncing.go | 2 +- p2p/discovery/option.go | 4 +++- p2p/stream/common/streammanager/cooldown.go | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/api/service/legacysync/epoch_syncing.go b/api/service/legacysync/epoch_syncing.go index 641fb58890..8d1cb37c4f 100644 --- a/api/service/legacysync/epoch_syncing.go +++ b/api/service/legacysync/epoch_syncing.go @@ -112,7 +112,7 @@ func syncLoop(bc core.BlockChain, syncConfig *SyncConfig) (timeout int) { } if otherEpoch < curEpoch { for _, peerCfg := range syncConfig.GetPeers() { - syncConfig.RemovePeer(peerCfg, fmt.Sprintf("[EPOCHSYNC]: current height is higher that others, remove peers: %s", peerCfg.String())) + syncConfig.RemovePeer(peerCfg, fmt.Sprintf("[EPOCHSYNC]: current height is higher than others, remove peers: %s", peerCfg.String())) } return 2 } diff --git a/p2p/discovery/option.go b/p2p/discovery/option.go index ce42592703..fff8eea20f 100644 --- a/p2p/discovery/option.go +++ b/p2p/discovery/option.go @@ -42,7 +42,9 @@ func (opt DHTConfig) GetLibp2pRawOptions() ([]libp2p_dht.Option, error) { opts = append(opts, libp2p_dht.Concurrency(opt.DiscConcurrency)) } - opts = append(opts, libp2p_dht.DisableAutoRefresh()) + // TODO: to disable auto refresh to make sure there is no conflicts with protocol discovery functions + // it's not applicable for legacy sync + // opts = append(opts, libp2p_dht.DisableAutoRefresh()) return opts, nil } diff --git a/p2p/stream/common/streammanager/cooldown.go b/p2p/stream/common/streammanager/cooldown.go index f22f4956f0..ef6d21bc98 100644 --- a/p2p/stream/common/streammanager/cooldown.go +++ b/p2p/stream/common/streammanager/cooldown.go @@ -36,6 +36,8 @@ func (cache *coolDownCache) Has(id peer.ID) bool { // Add adds the peer ID to the cache func (cache *coolDownCache) Add(id peer.ID) { + cache.mu.Lock() + defer cache.mu.Unlock() has := cache.timeCache.Has(string(id)) if !has { cache.timeCache.Add(string(id)) From d46aaf59d4448bf973f87a78f7d9a84d43171ae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Mon, 13 Feb 2023 15:24:35 +0800 Subject: [PATCH 194/420] add watermark low/high options for p2p connection manager --- cmd/harmony/config_migrations.go | 6 +++ cmd/harmony/default.go | 2 + cmd/harmony/flags.go | 20 +++++++++ cmd/harmony/flags_test.go | 12 ++++++ cmd/harmony/main.go | 2 + internal/configs/harmony/harmony.go | 2 + internal/configs/node/network.go | 8 +++- node/node_handler.go | 2 + p2p/discovery/option_test.go | 4 +- p2p/host.go | 64 +++++++++++++++++++---------- rosetta/infra/harmony-mainnet.conf | 2 + rosetta/infra/harmony-pstn.conf | 2 + 12 files changed, 101 insertions(+), 25 deletions(-) diff --git a/cmd/harmony/config_migrations.go b/cmd/harmony/config_migrations.go index 3a256f49d2..bdbfefdc91 100644 --- a/cmd/harmony/config_migrations.go +++ b/cmd/harmony/config_migrations.go @@ -319,6 +319,12 @@ func init() { } migrations["2.5.10"] = func(confTree *toml.Tree) *toml.Tree { + if confTree.Get("P2P.ConnManagerLowWatermark") == nil { + confTree.Set("P2P.ConnManagerLowWatermark", defaultConfig.P2P.ConnManagerLowWatermark) + } + if confTree.Get("P2P.ConnManagerHighWatermark") == nil { + confTree.Set("P2P.ConnManagerHighWatermark", defaultConfig.P2P.ConnManagerHighWatermark) + } if confTree.Get("Sync.MaxAdvertiseWaitTime") == nil { confTree.Set("Sync.MaxAdvertiseWaitTime", defaultConfig.Sync.MaxAdvertiseWaitTime) } diff --git a/cmd/harmony/default.go b/cmd/harmony/default.go index 2a6c7a75c3..78d2643b60 100644 --- a/cmd/harmony/default.go +++ b/cmd/harmony/default.go @@ -32,6 +32,8 @@ var defaultConfig = harmonyconfig.HarmonyConfig{ MaxConnsPerIP: nodeconfig.DefaultMaxConnPerIP, DisablePrivateIPScan: false, MaxPeers: nodeconfig.DefaultMaxPeers, + ConnManagerLowWatermark: nodeconfig.DefaultConnManagerLowWatermark, + ConnManagerHighWatermark: nodeconfig.DefaultConnManagerHighWatermark, WaitForEachPeerToConnect: nodeconfig.DefaultWaitForEachPeerToConnect, }, HTTP: harmonyconfig.HttpConfig{ diff --git a/cmd/harmony/flags.go b/cmd/harmony/flags.go index 168081660c..55da51f551 100644 --- a/cmd/harmony/flags.go +++ b/cmd/harmony/flags.go @@ -63,6 +63,8 @@ var ( p2pDisablePrivateIPScanFlag, maxConnPerIPFlag, maxPeersFlag, + connManagerLowWatermarkFlag, + connManagerHighWatermarkFlag, } httpFlags = []cli.Flag{ @@ -579,6 +581,16 @@ var ( Usage: "maximum number of peers allowed, 0 means no limit", DefValue: defaultConfig.P2P.MaxConnsPerIP, } + connManagerLowWatermarkFlag = cli.IntFlag{ + Name: "p2p.connmgr-low", + Usage: "lowest number of connections that'll be maintained in connection manager", + DefValue: defaultConfig.P2P.ConnManagerLowWatermark, + } + connManagerHighWatermarkFlag = cli.IntFlag{ + Name: "p2p.connmgr-high", + Usage: "highest number of connections that'll be maintained in connection manager", + DefValue: defaultConfig.P2P.ConnManagerHighWatermark, + } waitForEachPeerToConnectFlag = cli.BoolFlag{ Name: "p2p.wait-for-connections", Usage: "node waits for each single peer to connect and it doesn't add them to peers list after timeout", @@ -624,6 +636,14 @@ func applyP2PFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { config.P2P.WaitForEachPeerToConnect = cli.GetBoolFlagValue(cmd, waitForEachPeerToConnectFlag) } + if cli.IsFlagChanged(cmd, connManagerLowWatermarkFlag) { + config.P2P.ConnManagerLowWatermark = cli.GetIntFlagValue(cmd, connManagerLowWatermarkFlag) + } + + if cli.IsFlagChanged(cmd, connManagerHighWatermarkFlag) { + config.P2P.ConnManagerHighWatermark = cli.GetIntFlagValue(cmd, connManagerHighWatermarkFlag) + } + if cli.IsFlagChanged(cmd, p2pDisablePrivateIPScanFlag) { config.P2P.DisablePrivateIPScan = cli.GetBoolFlagValue(cmd, p2pDisablePrivateIPScanFlag) } diff --git a/cmd/harmony/flags_test.go b/cmd/harmony/flags_test.go index 7bd3e71999..9ebae4bb3e 100644 --- a/cmd/harmony/flags_test.go +++ b/cmd/harmony/flags_test.go @@ -65,6 +65,8 @@ func TestHarmonyFlags(t *testing.T) { MaxConnsPerIP: 5, DisablePrivateIPScan: false, MaxPeers: defaultConfig.P2P.MaxPeers, + ConnManagerLowWatermark: defaultConfig.P2P.ConnManagerLowWatermark, + ConnManagerHighWatermark: defaultConfig.P2P.ConnManagerHighWatermark, WaitForEachPeerToConnect: false, }, HTTP: harmonyconfig.HttpConfig{ @@ -374,6 +376,8 @@ func TestP2PFlags(t *testing.T) { MaxConnsPerIP: 10, DisablePrivateIPScan: false, MaxPeers: defaultConfig.P2P.MaxPeers, + ConnManagerLowWatermark: defaultConfig.P2P.ConnManagerLowWatermark, + ConnManagerHighWatermark: defaultConfig.P2P.ConnManagerHighWatermark, WaitForEachPeerToConnect: false, }, }, @@ -386,6 +390,8 @@ func TestP2PFlags(t *testing.T) { MaxConnsPerIP: 10, DisablePrivateIPScan: false, MaxPeers: defaultConfig.P2P.MaxPeers, + ConnManagerLowWatermark: defaultConfig.P2P.ConnManagerLowWatermark, + ConnManagerHighWatermark: defaultConfig.P2P.ConnManagerHighWatermark, WaitForEachPeerToConnect: false, }, }, @@ -399,6 +405,8 @@ func TestP2PFlags(t *testing.T) { MaxConnsPerIP: 5, DisablePrivateIPScan: false, MaxPeers: defaultConfig.P2P.MaxPeers, + ConnManagerLowWatermark: defaultConfig.P2P.ConnManagerLowWatermark, + ConnManagerHighWatermark: defaultConfig.P2P.ConnManagerHighWatermark, WaitForEachPeerToConnect: false, }, }, @@ -412,6 +420,8 @@ func TestP2PFlags(t *testing.T) { MaxConnsPerIP: nodeconfig.DefaultMaxConnPerIP, DisablePrivateIPScan: true, MaxPeers: defaultConfig.P2P.MaxPeers, + ConnManagerLowWatermark: defaultConfig.P2P.ConnManagerLowWatermark, + ConnManagerHighWatermark: defaultConfig.P2P.ConnManagerHighWatermark, WaitForEachPeerToConnect: false, }, }, @@ -425,6 +435,8 @@ func TestP2PFlags(t *testing.T) { MaxConnsPerIP: nodeconfig.DefaultMaxConnPerIP, DisablePrivateIPScan: defaultConfig.P2P.DisablePrivateIPScan, MaxPeers: 100, + ConnManagerLowWatermark: defaultConfig.P2P.ConnManagerLowWatermark, + ConnManagerHighWatermark: defaultConfig.P2P.ConnManagerHighWatermark, WaitForEachPeerToConnect: false, }, }, diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index b68e757422..eacc392f10 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -645,6 +645,8 @@ func createGlobalConfig(hc harmonyconfig.HarmonyConfig) (*nodeconfig.ConfigType, MaxConnPerIP: hc.P2P.MaxConnsPerIP, DisablePrivateIPScan: hc.P2P.DisablePrivateIPScan, MaxPeers: hc.P2P.MaxPeers, + ConnManagerLowWatermark: hc.P2P.ConnManagerLowWatermark, + ConnManagerHighWatermark: hc.P2P.ConnManagerHighWatermark, WaitForEachPeerToConnect: hc.P2P.WaitForEachPeerToConnect, ForceReachabilityPublic: forceReachabilityPublic, }) diff --git a/internal/configs/harmony/harmony.go b/internal/configs/harmony/harmony.go index 73ddd442dd..af3c8c9b61 100644 --- a/internal/configs/harmony/harmony.go +++ b/internal/configs/harmony/harmony.go @@ -54,6 +54,8 @@ type P2pConfig struct { MaxConnsPerIP int DisablePrivateIPScan bool MaxPeers int64 + ConnManagerLowWatermark int + ConnManagerHighWatermark int WaitForEachPeerToConnect bool } diff --git a/internal/configs/node/network.go b/internal/configs/node/network.go index 332b5cce74..8b15d3359c 100644 --- a/internal/configs/node/network.go +++ b/internal/configs/node/network.go @@ -63,7 +63,13 @@ const ( DefaultMaxConnPerIP = 10 // DefaultMaxPeers is the maximum number of remote peers, with 0 representing no limit DefaultMaxPeers = 0 - // DefaultWaitForEachPeerToConnect sets the sync configs to connect to neighbor peers one by one and waits for each peer to connect + // DefaultConnManagerLowWatermark is the lowest number of connections that'll be maintained in connection manager + DefaultConnManagerLowWatermark = 160 + // DefaultConnManagerHighWatermark is the highest number of connections that'll be maintained in connection manager + // When the peer count exceeds the 'high watermark', as many peers will be pruned (and + // their connections terminated) until 'low watermark' peers remain. + DefaultConnManagerHighWatermark = 192 + // DefaultWaitForEachPeerToConnect sets the sync configs to connect to neighbor peers one by one and waits for each peer to connect. DefaultWaitForEachPeerToConnect = false ) diff --git a/node/node_handler.go b/node/node_handler.go index 3db4f8deaf..a34d73e94d 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -3,6 +3,7 @@ package node import ( "bytes" "context" + "fmt" "math/rand" "time" @@ -447,6 +448,7 @@ func (node *Node) BootstrapConsensus() error { if numPeersNow >= min { utils.Logger().Info().Msg("[bootstrap] StartConsensus") enoughMinPeers <- struct{}{} + fmt.Println("Bootstrap consensus done.", numPeersNow, " peers are connected") return } utils.Logger().Info(). diff --git a/p2p/discovery/option_test.go b/p2p/discovery/option_test.go index e782151f59..1829e77c85 100644 --- a/p2p/discovery/option_test.go +++ b/p2p/discovery/option_test.go @@ -40,14 +40,14 @@ func TestDHTOption_getLibp2pRawOptions(t *testing.T) { opt: DHTConfig{ BootNodes: testAddrStr, }, - expLen: 2, + expLen: 1, }, { opt: DHTConfig{ BootNodes: testAddrStr, DataStoreFile: &validPath, }, - expLen: 3, + expLen: 2, }, { opt: DHTConfig{ diff --git a/p2p/host.go b/p2p/host.go index bab1da691c..9395e1df45 100644 --- a/p2p/host.go +++ b/p2p/host.go @@ -96,6 +96,8 @@ type HostConfig struct { MaxConnPerIP int DisablePrivateIPScan bool MaxPeers int64 + ConnManagerLowWatermark int + ConnManagerHighWatermark int WaitForEachPeerToConnect bool ForceReachabilityPublic bool } @@ -114,19 +116,6 @@ func init() { libp2p_pubsub.GossipSubMaxIHaveLength = 1000 } -func forceReachabilityPublic(f bool) libp2p_config.Option { - if f { - return func(cfg *libp2p_config.Config) error { - public := libp2p_network.Reachability(libp2p_network.ReachabilityPublic) - cfg.AutoNATConfig.ForceReachability = &public - return nil - } - } - return func(p2pConfig *libp2p_config.Config) error { - return nil - } -} - // NewHost .. func NewHost(cfg HostConfig) (Host, error) { var ( @@ -141,16 +130,20 @@ func NewHost(cfg HostConfig) (Host, error) { ) ctx, cancel := context.WithCancel(context.Background()) - // TODO: move low and high to configs - connmgr, err := connmgr.NewConnManager( - int(cfg.MaxConnPerIP), // LowWater - int(1024)*cfg.MaxConnPerIP, // HighWater, - connmgr.WithGracePeriod(time.Minute), - ) - if err != nil { + + // create connection manager + low := cfg.ConnManagerLowWatermark + high := cfg.ConnManagerHighWatermark + if high < low { cancel() - return nil, err + utils.Logger().Error(). + Int("low", cfg.ConnManagerLowWatermark). + Int("high", cfg.ConnManagerHighWatermark). + Msg("connection manager watermarks are invalid") + return nil, errors.New("invalid connection manager watermarks") } + + // prepare host options var idht *dht.IpfsDHT var opt discovery.DHTConfig p2pHostConfig := []libp2p.Option{ @@ -164,7 +157,7 @@ func NewHost(cfg HostConfig) (Host, error) { libp2p.DefaultTransports, // Prevent the peer from having too many // connections by attaching a connection manager. - libp2p.ConnectionManager(connmgr), + connectionManager(low, high), // Attempt to open ports using uPNP for NATed hosts. libp2p.NATPortMap(), libp2p.Routing(func(h host.Host) (routing.PeerRouting, error) { @@ -286,6 +279,33 @@ func NewHost(cfg HostConfig) (Host, error) { return h, nil } +// connectionManager creates a new connection manager and configures libp2p to use the +// given connection manager. +// lo and hi are watermarks governing the number of connections that'll be maintained. +// When the peer count exceeds the 'high watermark', as many peers will be pruned (and +// their connections terminated) until 'low watermark' peers remain. +func connectionManager(low int, high int) libp2p_config.Option { + if low > 0 && high > low { + connmgr, err := connmgr.NewConnManager( + low, // Low Watermark + high, // High Watermark + connmgr.WithGracePeriod(time.Minute), + ) + if err != nil { + utils.Logger().Error(). + Err(err). + Int("low", low). + Int("high", high). + Msg("create connection manager failed") + return nil + } + return libp2p.ConnectionManager(connmgr) + } + return func(p2pConfig *libp2p_config.Config) error { + return nil + } +} + // HostV2 is the version 2 p2p host type HostV2 struct { h libp2p_host.Host diff --git a/rosetta/infra/harmony-mainnet.conf b/rosetta/infra/harmony-mainnet.conf index 3e7085ba63..64e78c7eb2 100644 --- a/rosetta/infra/harmony-mainnet.conf +++ b/rosetta/infra/harmony-mainnet.conf @@ -68,6 +68,8 @@ Version = "2.5.11" MaxConnsPerIP = 10 MaxPeers = 0 Port = 9000 + ConnManagerLowWatermark = 160 + ConnManagerHighWatermark = 192 WaitForEachPeerToConnect = false [Pprof] diff --git a/rosetta/infra/harmony-pstn.conf b/rosetta/infra/harmony-pstn.conf index 458e826354..f9ded4d38c 100644 --- a/rosetta/infra/harmony-pstn.conf +++ b/rosetta/infra/harmony-pstn.conf @@ -68,6 +68,8 @@ Version = "2.5.11" MaxConnsPerIP = 10 MaxPeers = 0 Port = 9000 + ConnManagerLowWatermark = 160 + ConnManagerHighWatermark = 192 WaitForEachPeerToConnect = false [Pprof] From 047d10461dfa79b7afbf36d4ed0d67030824fdcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Mon, 13 Feb 2023 15:33:27 +0800 Subject: [PATCH 195/420] add test for new conn manager flags --- cmd/harmony/flags_test.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/cmd/harmony/flags_test.go b/cmd/harmony/flags_test.go index 9ebae4bb3e..7f8cf34a83 100644 --- a/cmd/harmony/flags_test.go +++ b/cmd/harmony/flags_test.go @@ -440,6 +440,36 @@ func TestP2PFlags(t *testing.T) { WaitForEachPeerToConnect: false, }, }, + { + args: []string{"--p2p.connmgr-low", "100"}, + expConfig: harmonyconfig.P2pConfig{ + Port: nodeconfig.DefaultP2PPort, + IP: nodeconfig.DefaultPublicListenIP, + KeyFile: "./.hmykey", + DiscConcurrency: nodeconfig.DefaultP2PConcurrency, + MaxConnsPerIP: nodeconfig.DefaultMaxConnPerIP, + DisablePrivateIPScan: defaultConfig.P2P.DisablePrivateIPScan, + MaxPeers: defaultConfig.P2P.MaxPeers, + ConnManagerLowWatermark: 100, + ConnManagerHighWatermark: defaultConfig.P2P.ConnManagerHighWatermark, + WaitForEachPeerToConnect: false, + }, + }, + { + args: []string{"--p2p.connmgr-high", "400"}, + expConfig: harmonyconfig.P2pConfig{ + Port: nodeconfig.DefaultP2PPort, + IP: nodeconfig.DefaultPublicListenIP, + KeyFile: "./.hmykey", + DiscConcurrency: nodeconfig.DefaultP2PConcurrency, + MaxConnsPerIP: nodeconfig.DefaultMaxConnPerIP, + DisablePrivateIPScan: defaultConfig.P2P.DisablePrivateIPScan, + MaxPeers: defaultConfig.P2P.MaxPeers, + ConnManagerLowWatermark: defaultConfig.P2P.ConnManagerLowWatermark, + ConnManagerHighWatermark: 400, + WaitForEachPeerToConnect: false, + }, + }, } for i, test := range tests { ts := newFlagTestSuite(t, append(p2pFlags, legacyMiscFlags...), From 26fd2d42177e341ead1398df60b134e9236b6720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Tue, 14 Feb 2023 08:09:08 +0800 Subject: [PATCH 196/420] fix dedent --- rosetta/infra/harmony-mainnet.conf | 2 +- rosetta/infra/harmony-pstn.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rosetta/infra/harmony-mainnet.conf b/rosetta/infra/harmony-mainnet.conf index 64e78c7eb2..8dc7909e3d 100644 --- a/rosetta/infra/harmony-mainnet.conf +++ b/rosetta/infra/harmony-mainnet.conf @@ -69,7 +69,7 @@ Version = "2.5.11" MaxPeers = 0 Port = 9000 ConnManagerLowWatermark = 160 - ConnManagerHighWatermark = 192 + ConnManagerHighWatermark = 192 WaitForEachPeerToConnect = false [Pprof] diff --git a/rosetta/infra/harmony-pstn.conf b/rosetta/infra/harmony-pstn.conf index f9ded4d38c..0835f70d78 100644 --- a/rosetta/infra/harmony-pstn.conf +++ b/rosetta/infra/harmony-pstn.conf @@ -69,7 +69,7 @@ Version = "2.5.11" MaxPeers = 0 Port = 9000 ConnManagerLowWatermark = 160 - ConnManagerHighWatermark = 192 + ConnManagerHighWatermark = 192 WaitForEachPeerToConnect = false [Pprof] From b13a5aaad994d02b265f1e3c996e3644ceb0af38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Tue, 14 Feb 2023 09:07:08 +0800 Subject: [PATCH 197/420] add comment to inform about p2p connection manager options --- cmd/harmony/flags.go | 4 ++-- internal/configs/harmony/harmony.go | 19 +++++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/cmd/harmony/flags.go b/cmd/harmony/flags.go index 55da51f551..947adacc5e 100644 --- a/cmd/harmony/flags.go +++ b/cmd/harmony/flags.go @@ -583,12 +583,12 @@ var ( } connManagerLowWatermarkFlag = cli.IntFlag{ Name: "p2p.connmgr-low", - Usage: "lowest number of connections that'll be maintained in connection manager", + Usage: "lowest number of connections that'll be maintained in connection manager. Set both high and low watermarks to zero to disable connection manager", DefValue: defaultConfig.P2P.ConnManagerLowWatermark, } connManagerHighWatermarkFlag = cli.IntFlag{ Name: "p2p.connmgr-high", - Usage: "highest number of connections that'll be maintained in connection manager", + Usage: "highest number of connections that'll be maintained in connection manager. Set both high and low watermarks to zero to disable connection manager", DefValue: defaultConfig.P2P.ConnManagerHighWatermark, } waitForEachPeerToConnectFlag = cli.BoolFlag{ diff --git a/internal/configs/harmony/harmony.go b/internal/configs/harmony/harmony.go index af3c8c9b61..c02f3bb7e0 100644 --- a/internal/configs/harmony/harmony.go +++ b/internal/configs/harmony/harmony.go @@ -46,14 +46,17 @@ type NetworkConfig struct { } type P2pConfig struct { - Port int - IP string - KeyFile string - DHTDataStore *string `toml:",omitempty"` - DiscConcurrency int // Discovery Concurrency value - MaxConnsPerIP int - DisablePrivateIPScan bool - MaxPeers int64 + Port int + IP string + KeyFile string + DHTDataStore *string `toml:",omitempty"` + DiscConcurrency int // Discovery Concurrency value + MaxConnsPerIP int + DisablePrivateIPScan bool + MaxPeers int64 + // In order to disable Connection Manager, it only needs to + // set both the high and low watermarks to zero. In this way, + // using Connection Manager will be an optional feature. ConnManagerLowWatermark int ConnManagerHighWatermark int WaitForEachPeerToConnect bool From cd8609973a074e88a4b13fa5101bcc3fa87d61bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Fri, 3 Feb 2023 09:17:49 +0800 Subject: [PATCH 198/420] fix max height issue --- api/service/legacysync/epoch_syncing.go | 22 +++++++++--------- api/service/legacysync/helpers.go | 9 ++++++-- api/service/legacysync/syncing.go | 30 +++++++++++++++++++------ api/service/stagedsync/stagedsync.go | 19 +++++++++++----- api/service/stagedsync/sync_status.go | 5 ++++- 5 files changed, 59 insertions(+), 26 deletions(-) diff --git a/api/service/legacysync/epoch_syncing.go b/api/service/legacysync/epoch_syncing.go index 8d1cb37c4f..eefca9a5cb 100644 --- a/api/service/legacysync/epoch_syncing.go +++ b/api/service/legacysync/epoch_syncing.go @@ -2,7 +2,6 @@ package legacysync import ( "fmt" - "math" "sync" "time" @@ -49,11 +48,12 @@ func (ss *EpochSync) isSynchronized(_ bool) SyncCheckResult { if ss.syncConfig == nil { return SyncCheckResult{} // If syncConfig is not instantiated, return not in sync } - otherHeight1 := getMaxPeerHeight(ss.syncConfig) - if otherHeight1 == math.MaxUint64 { + otherHeight1, errMaxHeight := getMaxPeerHeight(ss.syncConfig) + if errMaxHeight != nil { utils.Logger().Error(). Uint64("OtherHeight", otherHeight1). Int("Peers count", ss.syncConfig.PeersCount()). + Err(errMaxHeight). Msg("[EPOCHSYNC] No peers for get height") return SyncCheckResult{} } @@ -93,15 +93,15 @@ func (ss *EpochSync) SyncLoop(bc core.BlockChain, consensus *consensus.Consensus func syncLoop(bc core.BlockChain, syncConfig *SyncConfig) (timeout int) { isBeacon := bc.ShardID() == 0 - maxHeight := getMaxPeerHeight(syncConfig) - for { - if maxHeight == 0 || maxHeight == math.MaxUint64 { - utils.Logger().Info(). - Msgf("[EPOCHSYNC] No peers to sync (isBeacon: %t, ShardID: %d, peersCount: %d)", - isBeacon, bc.ShardID(), syncConfig.PeersCount()) - return 10 - } + maxHeight, errMaxHeight := getMaxPeerHeight(syncConfig) + if errMaxHeight != nil { + utils.Logger().Info(). + Msgf("[EPOCHSYNC] No peers to sync (isBeacon: %t, ShardID: %d, peersCount: %d)", + isBeacon, bc.ShardID(), syncConfig.PeersCount()) + return 10 + } + for { curEpoch := bc.CurrentBlock().Epoch().Uint64() otherEpoch := shard.Schedule.CalcEpochNumber(maxHeight).Uint64() if otherEpoch == curEpoch+1 { diff --git a/api/service/legacysync/helpers.go b/api/service/legacysync/helpers.go index 4699257f8b..ae1fc34f16 100644 --- a/api/service/legacysync/helpers.go +++ b/api/service/legacysync/helpers.go @@ -14,7 +14,7 @@ import ( ) // getMaxPeerHeight gets the maximum blockchain heights from peers -func getMaxPeerHeight(syncConfig *SyncConfig) uint64 { +func getMaxPeerHeight(syncConfig *SyncConfig) (uint64, error) { maxHeight := uint64(math.MaxUint64) var ( wg sync.WaitGroup @@ -49,7 +49,12 @@ func getMaxPeerHeight(syncConfig *SyncConfig) uint64 { return }) wg.Wait() - return maxHeight + + if maxHeight == uint64(math.MaxUint64) { + return 0, fmt.Errorf("[SYNC] get max peer height failed") + } + + return maxHeight, nil } func createSyncConfig(syncConfig *SyncConfig, peers []p2p.Peer, shardID uint32, selfPeerID libp2p_peer.ID, waitForEachPeerToConnect bool) (*SyncConfig, error) { diff --git a/api/service/legacysync/syncing.go b/api/service/legacysync/syncing.go index 0f004bfb4f..51366ab94c 100644 --- a/api/service/legacysync/syncing.go +++ b/api/service/legacysync/syncing.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "fmt" + "math" "math/rand" "reflect" "sort" @@ -1060,13 +1061,16 @@ func (ss *StateSync) RegisterNodeInfo() int { // IsSameBlockchainHeight checks whether the node is out of sync from other peers func (ss *StateSync) IsSameBlockchainHeight(bc core.BlockChain) (uint64, bool) { - otherHeight := getMaxPeerHeight(ss.syncConfig) + otherHeight, err := getMaxPeerHeight(ss.syncConfig) + if err != nil { + return 0, false + } currentHeight := bc.CurrentBlock().NumberU64() return otherHeight, currentHeight == otherHeight } // GetMaxPeerHeight .. -func (ss *StateSync) GetMaxPeerHeight() uint64 { +func (ss *StateSync) GetMaxPeerHeight() (uint64, error) { return getMaxPeerHeight(ss.syncConfig) } @@ -1079,9 +1083,9 @@ func (ss *StateSync) SyncLoop(bc core.BlockChain, worker *worker.Worker, isBeaco for { start := time.Now() - otherHeight := getMaxPeerHeight(ss.syncConfig) currentHeight := bc.CurrentBlock().NumberU64() - if currentHeight >= otherHeight { + otherHeight, errMaxHeight := getMaxPeerHeight(ss.syncConfig) + if errMaxHeight != nil || currentHeight >= otherHeight { utils.Logger().Info(). Msgf("[SYNC] Node is now IN SYNC! (isBeacon: %t, ShardID: %d, otherHeight: %d, currentHeight: %d)", isBeacon, bc.ShardID(), otherHeight, currentHeight) @@ -1218,7 +1222,9 @@ func (status *syncStatus) Get(fallback func() SyncCheckResult) SyncCheckResult { defer status.lock.Unlock() if status.expired() { result := fallback() - status.update(result) + if result.OtherHeight > 0 && result.OtherHeight < uint64(math.MaxUint64) { + status.update(result) + } } return status.lastResult } @@ -1280,8 +1286,15 @@ func (ss *StateSync) isSynchronized(doubleCheck bool) SyncCheckResult { if ss.syncConfig == nil { return SyncCheckResult{} // If syncConfig is not instantiated, return not in sync } - otherHeight1 := getMaxPeerHeight(ss.syncConfig) lastHeight := ss.blockChain.CurrentBlock().NumberU64() + otherHeight1, errMaxHeight1 := getMaxPeerHeight(ss.syncConfig) + if errMaxHeight1 != nil { + return SyncCheckResult{ + IsSynchronized: false, + OtherHeight: 0, + HeightDiff: 0, + } + } wasOutOfSync := lastHeight+inSyncThreshold < otherHeight1 if !doubleCheck { @@ -1302,7 +1315,10 @@ func (ss *StateSync) isSynchronized(doubleCheck bool) SyncCheckResult { // double check the sync status after 1 second to confirm (avoid false alarm) time.Sleep(1 * time.Second) - otherHeight2 := getMaxPeerHeight(ss.syncConfig) + otherHeight2, errMaxHeight2 := getMaxPeerHeight(ss.syncConfig) + if errMaxHeight2 != nil { + otherHeight2 = otherHeight1 + } currentHeight := ss.blockChain.CurrentBlock().NumberU64() isOutOfSync := currentHeight+inSyncThreshold < otherHeight2 diff --git a/api/service/stagedsync/stagedsync.go b/api/service/stagedsync/stagedsync.go index c81f0b5b12..aca707f619 100644 --- a/api/service/stagedsync/stagedsync.go +++ b/api/service/stagedsync/stagedsync.go @@ -1201,9 +1201,8 @@ func (ss *StagedSync) IsSameBlockchainHeight(bc core.BlockChain) (uint64, bool) } // GetMaxPeerHeight returns maximum block height of connected peers -func (ss *StagedSync) GetMaxPeerHeight() uint64 { - mph, _ := ss.getMaxPeerHeight() - return mph +func (ss *StagedSync) GetMaxPeerHeight() (uint64, error) { + return ss.getMaxPeerHeight() } func (ss *StagedSync) addConsensusLastMile(bc core.BlockChain, consensus *consensus.Consensus) error { @@ -1277,8 +1276,15 @@ func (ss *StagedSync) isSynchronized(doubleCheck bool) SyncCheckResult { if ss.syncConfig == nil { return SyncCheckResult{} // If syncConfig is not instantiated, return not in sync } - otherHeight1, _ := ss.getMaxPeerHeight() lastHeight := ss.Blockchain().CurrentBlock().NumberU64() + otherHeight1, errMaxHeight1 := ss.getMaxPeerHeight() + if errMaxHeight1 != nil { + return SyncCheckResult{ + IsSynchronized: false, + OtherHeight: 0, + HeightDiff: 0, + } + } wasOutOfSync := lastHeight+inSyncThreshold < otherHeight1 if !doubleCheck { @@ -1299,7 +1305,10 @@ func (ss *StagedSync) isSynchronized(doubleCheck bool) SyncCheckResult { // double check the sync status after 1 second to confirm (avoid false alarm) time.Sleep(1 * time.Second) - otherHeight2, _ := ss.getMaxPeerHeight() + otherHeight2, errMaxHeight2 := ss.getMaxPeerHeight() + if errMaxHeight2 != nil { + otherHeight2 = otherHeight1 + } currentHeight := ss.Blockchain().CurrentBlock().NumberU64() isOutOfSync := currentHeight+inSyncThreshold < otherHeight2 diff --git a/api/service/stagedsync/sync_status.go b/api/service/stagedsync/sync_status.go index 556f1058ba..3f1bf11e89 100644 --- a/api/service/stagedsync/sync_status.go +++ b/api/service/stagedsync/sync_status.go @@ -1,6 +1,7 @@ package stagedsync import ( + "math" "sync" "time" @@ -75,7 +76,9 @@ func (status *syncStatus) Get(fallback func() SyncCheckResult) SyncCheckResult { defer status.lock.Unlock() if status.expired() { result := fallback() - status.update(result) + if result.OtherHeight > 0 && result.OtherHeight < uint64(math.MaxUint64) { + status.update(result) + } } return status.lastResult } From f5c95a6a6e13dc334851e21be5028936145bf4e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Fri, 3 Feb 2023 17:04:14 +0800 Subject: [PATCH 199/420] add a separate log for get max height error --- api/service/legacysync/syncing.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/api/service/legacysync/syncing.go b/api/service/legacysync/syncing.go index 51366ab94c..43f07c039b 100644 --- a/api/service/legacysync/syncing.go +++ b/api/service/legacysync/syncing.go @@ -1085,7 +1085,16 @@ func (ss *StateSync) SyncLoop(bc core.BlockChain, worker *worker.Worker, isBeaco start := time.Now() currentHeight := bc.CurrentBlock().NumberU64() otherHeight, errMaxHeight := getMaxPeerHeight(ss.syncConfig) - if errMaxHeight != nil || currentHeight >= otherHeight { + if errMaxHeight != nil { + utils.Logger().Error(). + Bool("isBeacon", isBeacon). + Uint32("ShardID", bc.ShardID()). + Uint64("currentHeight", currentHeight). + Int("peers count", ss.syncConfig.PeersCount()). + Msgf("[SYNC] get max height failed") + break + } + if currentHeight >= otherHeight { utils.Logger().Info(). Msgf("[SYNC] Node is now IN SYNC! (isBeacon: %t, ShardID: %d, otherHeight: %d, currentHeight: %d)", isBeacon, bc.ShardID(), otherHeight, currentHeight) From 267edfd29cdd479e74546497b939ae13ff13a8aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Fri, 10 Feb 2023 22:59:05 +0800 Subject: [PATCH 200/420] fix log --- api/service/legacysync/helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/service/legacysync/helpers.go b/api/service/legacysync/helpers.go index ae1fc34f16..14dac994d9 100644 --- a/api/service/legacysync/helpers.go +++ b/api/service/legacysync/helpers.go @@ -51,7 +51,7 @@ func getMaxPeerHeight(syncConfig *SyncConfig) (uint64, error) { wg.Wait() if maxHeight == uint64(math.MaxUint64) { - return 0, fmt.Errorf("[SYNC] get max peer height failed") + return 0, fmt.Errorf("get max peer height failed") } return maxHeight, nil From 6dfcd51ba3b44d3fac96bc9f6a4af27fcfa4f4b1 Mon Sep 17 00:00:00 2001 From: Sun Hyuk Ahn Date: Thu, 9 Feb 2023 16:46:46 -0800 Subject: [PATCH 201/420] feat: triesInMemory flag --- cmd/harmony/config_migrations.go | 8 +++ cmd/harmony/default.go | 3 +- cmd/harmony/flags.go | 14 +++++ cmd/harmony/flags_test.go | 88 +++++++++++++++++------------ core/blockchain_impl.go | 19 +++---- internal/configs/harmony/harmony.go | 1 + internal/shardchain/shardchains.go | 10 ++++ rosetta/infra/harmony-mainnet.conf | 1 + rosetta/infra/harmony-pstn.conf | 1 + 9 files changed, 97 insertions(+), 48 deletions(-) diff --git a/cmd/harmony/config_migrations.go b/cmd/harmony/config_migrations.go index bdbfefdc91..7c9420a0be 100644 --- a/cmd/harmony/config_migrations.go +++ b/cmd/harmony/config_migrations.go @@ -332,6 +332,14 @@ func init() { return confTree } + migrations["2.5.11"] = func(confTree *toml.Tree) *toml.Tree { + if confTree.Get("General.TriesInMemory") == nil { + confTree.Set("General.TriesInMemory", defaultConfig.General.TriesInMemory) + } + confTree.Set("Version", "2.5.12") + return confTree + } + // check that the latest version here is the same as in default.go largestKey := getNextVersion(migrations) if largestKey != tomlConfigVersion { diff --git a/cmd/harmony/default.go b/cmd/harmony/default.go index 78d2643b60..3835c49210 100644 --- a/cmd/harmony/default.go +++ b/cmd/harmony/default.go @@ -5,7 +5,7 @@ import ( nodeconfig "github.com/harmony-one/harmony/internal/configs/node" ) -const tomlConfigVersion = "2.5.11" +const tomlConfigVersion = "2.5.12" const ( defNetworkType = nodeconfig.Mainnet @@ -22,6 +22,7 @@ var defaultConfig = harmonyconfig.HarmonyConfig{ IsOffline: false, DataDir: "./", TraceEnable: false, + TriesInMemory: 128, }, Network: getDefaultNetworkConfig(defNetworkType), P2P: harmonyconfig.P2pConfig{ diff --git a/cmd/harmony/flags.go b/cmd/harmony/flags.go index 947adacc5e..c3b03e336f 100644 --- a/cmd/harmony/flags.go +++ b/cmd/harmony/flags.go @@ -31,6 +31,7 @@ var ( legacyDataDirFlag, taraceFlag, + triesInMemoryFlag, } dnsSyncFlags = []cli.Flag{ @@ -322,6 +323,11 @@ var ( Usage: "indicates if full transaction tracing should be enabled", DefValue: defaultConfig.General.TraceEnable, } + triesInMemoryFlag = cli.IntFlag{ + Name: "blockchain.tries_in_memory", + Usage: "number of blocks from header stored in disk before exiting", + DefValue: defaultConfig.General.TriesInMemory, + } ) func getRootFlags() []cli.Flag { @@ -399,6 +405,14 @@ func applyGeneralFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) if cli.IsFlagChanged(cmd, isBackupFlag) { config.General.IsBackup = cli.GetBoolFlagValue(cmd, isBackupFlag) } + + if cli.IsFlagChanged(cmd, triesInMemoryFlag) { + value := cli.GetIntFlagValue(cmd, triesInMemoryFlag) + if value <= 1 { + panic("Must number greater than 1 for txpool.accountslots") + } + config.General.TriesInMemory = value + } } // network flags diff --git a/cmd/harmony/flags_test.go b/cmd/harmony/flags_test.go index 7f8cf34a83..15ba9aaf08 100644 --- a/cmd/harmony/flags_test.go +++ b/cmd/harmony/flags_test.go @@ -35,11 +35,12 @@ func TestHarmonyFlags(t *testing.T) { expConfig: harmonyconfig.HarmonyConfig{ Version: tomlConfigVersion, General: harmonyconfig.GeneralConfig{ - NodeType: "validator", - NoStaking: false, - ShardID: -1, - IsArchival: false, - DataDir: "./", + NodeType: "validator", + NoStaking: false, + ShardID: -1, + IsArchival: false, + DataDir: "./", + TriesInMemory: 128, }, Network: harmonyconfig.NetworkConfig{ NetworkType: "mainnet", @@ -186,63 +187,80 @@ func TestGeneralFlags(t *testing.T) { { args: []string{}, expConfig: harmonyconfig.GeneralConfig{ - NodeType: "validator", - NoStaking: false, - ShardID: -1, - IsArchival: false, - DataDir: "./", + NodeType: "validator", + NoStaking: false, + ShardID: -1, + IsArchival: false, + DataDir: "./", + TriesInMemory: 128, }, }, { args: []string{"--run", "explorer", "--run.legacy", "--run.shard=0", "--run.archive=true", "--datadir=./.hmy"}, expConfig: harmonyconfig.GeneralConfig{ - NodeType: "explorer", - NoStaking: true, - ShardID: 0, - IsArchival: true, - DataDir: "./.hmy", + NodeType: "explorer", + NoStaking: true, + ShardID: 0, + IsArchival: true, + DataDir: "./.hmy", + TriesInMemory: 128, }, }, { args: []string{"--node_type", "explorer", "--staking", "--shard_id", "0", "--is_archival", "--db_dir", "./"}, expConfig: harmonyconfig.GeneralConfig{ - NodeType: "explorer", - NoStaking: false, - ShardID: 0, - IsArchival: true, - DataDir: "./", + NodeType: "explorer", + NoStaking: false, + ShardID: 0, + IsArchival: true, + DataDir: "./", + TriesInMemory: 128, }, }, { args: []string{"--staking=false", "--is_archival=false"}, expConfig: harmonyconfig.GeneralConfig{ - NodeType: "validator", - NoStaking: true, - ShardID: -1, - IsArchival: false, - DataDir: "./", + NodeType: "validator", + NoStaking: true, + ShardID: -1, + IsArchival: false, + DataDir: "./", + TriesInMemory: 128, }, }, { args: []string{"--run", "explorer", "--run.shard", "0"}, expConfig: harmonyconfig.GeneralConfig{ - NodeType: "explorer", - NoStaking: false, - ShardID: 0, - IsArchival: false, - DataDir: "./", + NodeType: "explorer", + NoStaking: false, + ShardID: 0, + IsArchival: false, + DataDir: "./", + TriesInMemory: 128, }, }, { args: []string{"--run", "explorer", "--run.shard", "0", "--run.archive=false"}, expConfig: harmonyconfig.GeneralConfig{ - NodeType: "explorer", - NoStaking: false, - ShardID: 0, - IsArchival: false, - DataDir: "./", + NodeType: "explorer", + NoStaking: false, + ShardID: 0, + IsArchival: false, + DataDir: "./", + TriesInMemory: 128, + }, + }, + { + args: []string{"--blockchain.tries_in_memory", "64"}, + expConfig: harmonyconfig.GeneralConfig{ + NodeType: "validator", + NoStaking: false, + ShardID: -1, + IsArchival: false, + DataDir: "./", + TriesInMemory: 64, }, }, } diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 489fb3661e..01f1b21e65 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -103,7 +103,6 @@ const ( maxFutureBlocks = 16 maxTimeFutureBlocks = 30 badBlockLimit = 10 - triesInMemory = 128 triesInRedis = 1000 shardCacheLimit = 10 commitsCacheLimit = 10 @@ -127,6 +126,7 @@ type CacheConfig struct { Disabled bool // Whether to disable trie write caching (archive node) TrieNodeLimit int // Memory limit (MB) at which to flush the current in-memory trie to disk TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk + TriesInMemory uint64 // Block number from the head stored in disk before exiting } type BlockChainImpl struct { @@ -223,12 +223,7 @@ func newBlockChainWithOptions( db ethdb.Database, stateCache state.Database, beaconChain BlockChain, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus_engine.Engine, vmConfig vm.Config, options Options) (*BlockChainImpl, error) { - if cacheConfig == nil { - cacheConfig = &CacheConfig{ - TrieNodeLimit: 256 * 1024 * 1024, - TrieTimeLimit: 2 * time.Minute, - } - } + bodyCache, _ := lru.New(bodyCacheLimit) bodyRLPCache, _ := lru.New(bodyCacheLimit) receiptsCache, _ := lru.New(receiptsCacheLimit) @@ -1080,7 +1075,7 @@ func (bc *BlockChainImpl) Stop() { if !bc.cacheConfig.Disabled { triedb := bc.stateCache.TrieDB() - for _, offset := range []uint64{0, 1, triesInMemory - 1} { + for _, offset := range []uint64{0, 1, bc.cacheConfig.TriesInMemory - 1} { if number := bc.CurrentBlock().NumberU64(); number > offset { recent := bc.GetHeaderByNumber(number - offset) if recent != nil { @@ -1407,7 +1402,7 @@ func (bc *BlockChainImpl) WriteBlockWithState( triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive bc.triegc.Push(root, -int64(block.NumberU64())) - if current := block.NumberU64(); current > triesInMemory { + if current := block.NumberU64(); current > bc.cacheConfig.TriesInMemory { // If we exceeded our memory allowance, flush matured singleton nodes to disk var ( nodes, imgs = triedb.Size() @@ -1417,7 +1412,7 @@ func (bc *BlockChainImpl) WriteBlockWithState( triedb.Cap(limit - ethdb.IdealBatchSize) } // Find the next state trie we need to commit - header := bc.GetHeaderByNumber(current - triesInMemory) + header := bc.GetHeaderByNumber(current - bc.cacheConfig.TriesInMemory) if header != nil { chosen := header.Number().Uint64() @@ -1425,11 +1420,11 @@ func (bc *BlockChainImpl) WriteBlockWithState( if bc.gcproc > bc.cacheConfig.TrieTimeLimit { // If we're exceeding limits but haven't reached a large enough memory gap, // warn the user that the system is becoming unstable. - if chosen < lastWrite+triesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { + if chosen < lastWrite+bc.cacheConfig.TriesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { utils.Logger().Info(). Dur("time", bc.gcproc). Dur("allowance", bc.cacheConfig.TrieTimeLimit). - Float64("optimum", float64(chosen-lastWrite)/triesInMemory). + Float64("optimum", float64(chosen-lastWrite)/float64(bc.cacheConfig.TriesInMemory)). Msg("State in memory for too long, committing") } // Flush an entire trie and restart the counters diff --git a/internal/configs/harmony/harmony.go b/internal/configs/harmony/harmony.go index c02f3bb7e0..67c29f820f 100644 --- a/internal/configs/harmony/harmony.go +++ b/internal/configs/harmony/harmony.go @@ -74,6 +74,7 @@ type GeneralConfig struct { TraceEnable bool EnablePruneBeaconChain bool RunElasticMode bool + TriesInMemory int } type TiKVConfig struct { diff --git a/internal/shardchain/shardchains.go b/internal/shardchain/shardchains.go index 8f9b185956..1be2c68414 100644 --- a/internal/shardchain/shardchains.go +++ b/internal/shardchain/shardchains.go @@ -3,6 +3,7 @@ package shardchain import ( "math/big" "sync" + "time" "github.com/harmony-one/harmony/core/state" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" @@ -104,6 +105,15 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c utils.Logger().Info(). Uint32("shardID", shardID). Msg("disable cache, running in archival mode") + } else { + cacheConfig = &core.CacheConfig{ + TrieNodeLimit: 256 * 1024 * 1024, + TrieTimeLimit: 2 * time.Minute, + TriesInMemory: 128, + } + if sc.harmonyconfig != nil { + cacheConfig.TriesInMemory = uint64(sc.harmonyconfig.General.TriesInMemory) + } } chainConfig := *sc.chainConfig diff --git a/rosetta/infra/harmony-mainnet.conf b/rosetta/infra/harmony-mainnet.conf index 8dc7909e3d..a929eeeab1 100644 --- a/rosetta/infra/harmony-mainnet.conf +++ b/rosetta/infra/harmony-mainnet.conf @@ -35,6 +35,7 @@ Version = "2.5.11" RunElasticMode = false ShardID = 0 TraceEnable = false + TriesInMemory = 128 [HTTP] AuthPort = 9501 diff --git a/rosetta/infra/harmony-pstn.conf b/rosetta/infra/harmony-pstn.conf index 0835f70d78..edb911f87a 100644 --- a/rosetta/infra/harmony-pstn.conf +++ b/rosetta/infra/harmony-pstn.conf @@ -35,6 +35,7 @@ Version = "2.5.11" RunElasticMode = false ShardID = 0 TraceEnable = false + TriesInMemory = 128 [HTTP] AuthPort = 9501 From b51be8bea3dfd9d0a0897c99dc562b857e30593a Mon Sep 17 00:00:00 2001 From: Sun Hyuk Ahn Date: Fri, 10 Feb 2023 10:56:17 -0800 Subject: [PATCH 202/420] fix: panic if TriesInMemory is 1 to 2 --- cmd/harmony/flags.go | 4 ++-- core/blockchain_impl.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/harmony/flags.go b/cmd/harmony/flags.go index c3b03e336f..2ffceb6e5b 100644 --- a/cmd/harmony/flags.go +++ b/cmd/harmony/flags.go @@ -408,8 +408,8 @@ func applyGeneralFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) if cli.IsFlagChanged(cmd, triesInMemoryFlag) { value := cli.GetIntFlagValue(cmd, triesInMemoryFlag) - if value <= 1 { - panic("Must number greater than 1 for txpool.accountslots") + if value <= 2 { + panic("Must provide number greater than 2 for General.TriesInMemory") } config.General.TriesInMemory = value } diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 01f1b21e65..8b0683bd65 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -1071,7 +1071,7 @@ func (bc *BlockChainImpl) Stop() { // We're writing three different states to catch different restart scenarios: // - HEAD: So we don't need to reprocess any blocks in the general case // - HEAD-1: So we don't do large reorgs if our HEAD becomes an uncle - // - HEAD-127: So we have a hard limit on the number of blocks reexecuted + // - HEAD-TriesInMemory: So we have a configurable hard limit on the number of blocks reexecuted (default 128) if !bc.cacheConfig.Disabled { triedb := bc.stateCache.TrieDB() From 01c9aa13e5dbfee438c6781b9f56a440e426dd03 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 1 Sep 2022 00:53:16 +0700 Subject: [PATCH 203/420] in progress. --- api/service/explorer/service.go | 48 +++++++++++++++++++++++++++++++ cmd/harmony/main.go | 6 +++- consensus/consensus.go | 2 ++ consensus/consensus_service.go | 27 +++++++++++++++++ consensus/consensus_v2.go | 31 ++++++++++++++++++-- consensus/leader.go | 9 ++++++ consensus/quorum/quorum.go | 7 ++++- consensus/validator.go | 14 +++++++++ consensus/view_change.go | 19 ++++++++++++ hmy/hmy.go | 2 ++ node/api.go | 6 ++++ node/node_newblock.go | 4 +++ test/configs/local-resharding.txt | 31 ++++++-------------- test/deploy.sh | 5 +++- 14 files changed, 184 insertions(+), 27 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index b5ffa1b8b0..d96c1bc56c 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -7,11 +7,13 @@ import ( "fmt" "net" "net/http" + "os" "path" "strconv" "time" "github.com/RoaringBitmap/roaring/roaring64" + "github.com/harmony-one/harmony/internal/common" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" ethCommon "github.com/ethereum/go-ethereum/common" @@ -113,12 +115,17 @@ func (s *Service) Run() *http.Server { s.router = mux.NewRouter() + fmt.Println("++", addr) + // Set up router for addresses. // Fetch addresses request, accepts parameter size: how much addresses to read, // parameter prefix: from which address prefix start s.router.Path("/addresses").Queries("size", "{[0-9]*?}", "prefix", "{[a-zA-Z0-9]*?}").HandlerFunc(s.GetAddresses).Methods("GET") s.router.Path("/addresses").HandlerFunc(s.GetAddresses) s.router.Path("/height").HandlerFunc(s.GetHeight) + s.router.Path("/leader").HandlerFunc(s.GetLeader) + s.router.Path("/blocks").HandlerFunc(s.GetBlocks) + s.router.Path("/halt").HandlerFunc(s.halt) // Set up router for supply info s.router.Path("/burn-addresses").Queries().HandlerFunc(s.GetInaccessibleAddressInfo).Methods("GET") @@ -186,6 +193,47 @@ type HeightResponse struct { S3 uint64 `json:"3,omitempty"` } +func (s *Service) GetLeader(w http.ResponseWriter, r *http.Request) { + if s.backend.IsCurrentlyLeader() { + w.Write([]byte("true ")) + } else { + w.Write([]byte("false")) + } + + keys := "" + for _, p := range s.backend.GetPublicKeys() { + addr := common.Address{} + addrBytes := p.Object.GetAddress() + addr.SetBytes(addrBytes[:]) + keys += fmt.Sprintf("%s ", addr.String()) + break + } + //blsPubKeyBytes := leaderKey.Object.GetAddress() + //coinbase.SetBytes(blsPubKeyBytes[:]) + + w.Write([]byte(fmt.Sprintf(" %d", s.blockchain.ShardID()))) + w.Write([]byte(fmt.Sprintf(" %s", s.Port))) + w.Write([]byte(fmt.Sprintf(" %s", keys))) + w.Write([]byte(fmt.Sprintf(" %s", s.backend.GetPublicKeys().SerializeToHexStr()))) + +} + +func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { + cur := s.blockchain.CurrentHeader().Number().Uint64() + + for i := cur; i > 0; i-- { + block := s.blockchain.GetBlockByNumber(i) + w.Write([]byte(fmt.Sprintf("%d ", i))) + w.Write([]byte(fmt.Sprintf("%s ", block.Header().ViewID().String()))) + w.Write([]byte(fmt.Sprintf("%s ", block.Header().Coinbase().Hash().Hex()))) + w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) + } +} + +func (s *Service) halt(w http.ResponseWriter, r *http.Request) { + os.Exit(0) +} + // GetHeight returns heights of current and beacon chains if needed. func (s *Service) GetHeight(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index eacc392f10..2fcfe5f037 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -266,6 +266,8 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { var err error + fmt.Println("OS: ", os.Args) + nodeconfigSetShardSchedule(hc) nodeconfig.SetShardingSchedule(shard.Schedule) nodeconfig.SetVersion(getHarmonyVersion()) @@ -425,8 +427,9 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { if currentNode.NodeConfig.Role() == nodeconfig.Validator { currentNode.RegisterValidatorServices() } else if currentNode.NodeConfig.Role() == nodeconfig.ExplorerNode { - currentNode.RegisterExplorerServices() + } + currentNode.RegisterExplorerServices() currentNode.RegisterService(service.CrosslinkSending, crosslink_sending.New(currentNode, currentNode.Blockchain())) if hc.Pprof.Enabled { setupPprofService(currentNode, hc) @@ -800,6 +803,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // Set the consensus ID to be the current block number viewID := currentNode.Blockchain().CurrentBlock().Header().ViewID().Uint64() + fmt.Println("viewID:", viewID) currentConsensus.SetViewIDs(viewID + 1) utils.Logger().Info(). Uint64("viewID", viewID). diff --git a/consensus/consensus.go b/consensus/consensus.go index e0e096e306..0c51d648c0 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -73,6 +73,8 @@ type Consensus struct { priKey multibls.PrivateKeys // the publickey of leader LeaderPubKey *bls.PublicKeyWrapper + // index of leader in the list of validators. + LeaderIndex int // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e6be606f2e..e646504dde 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync/atomic" "time" @@ -212,6 +213,7 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } + fmt.Println("[checkViewID] Set LEADEER PUB KEY ", msg.SenderPubkeys[0].Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() @@ -395,6 +397,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { Str("leaderPubKey", leaderPubKey.Bytes.Hex()). Msg("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain") consensus.pubKeyLock.Lock() + fmt.Println("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain", leaderPubKey.Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = leaderPubKey consensus.pubKeyLock.Unlock() } @@ -461,6 +464,7 @@ func (consensus *Consensus) isLeader() bool { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { + fmt.Println("SetViewIDs", height) consensus.SetCurBlockViewID(height) consensus.SetViewChangingID(height) } @@ -470,6 +474,20 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } +// SetLeaderIndex set the leader index. +func (consensus *Consensus) SetLeaderIndex(f func(int) int) (current int) { + consensus.pubKeyLock.Lock() + defer consensus.pubKeyLock.Unlock() + consensus.LeaderIndex = f(consensus.LeaderIndex) + return consensus.LeaderIndex +} + +func (consensus *Consensus) GetLeaderIndex() int { + consensus.pubKeyLock.Lock() + defer consensus.pubKeyLock.Unlock() + return consensus.LeaderIndex +} + // SetViewChangingID set the current view change ID func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) @@ -480,6 +498,15 @@ func (consensus *Consensus) StartFinalityCount() { consensus.finalityCounter.Store(time.Now().UnixNano()) } +//func (consensus *Consensus) ReshardingNextLeader(newblock *types.Block) { +// consensus.pubKeyLock.Lock() +// fmt.Println("nextBlock1 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) +// consensus.LeaderPubKey = consensus.getNextLeaderKey(consensus.GetCurBlockViewID() + 1) +// fmt.Println("nextBlock2 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) +// consensus.pubKeyLock.Unlock() +// +//} + // FinishFinalityCount calculate the current finality func (consensus *Consensus) FinishFinalityCount() { d := time.Now().UnixNano() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 0b1ce28cf4..bf042c62d3 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/hex" + "fmt" "math/big" "sync/atomic" "time" @@ -131,6 +132,7 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb } func (consensus *Consensus) finalCommit() { + // THIS IS NOT GOOD PLACE FOR LEADER SWITCHING numCommits := consensus.Decider.SignersCount(quorum.Commit) consensus.getLogger().Info(). @@ -393,6 +395,7 @@ func (consensus *Consensus) Start( consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() case newBlock := <-blockChannel: + //consensus.ReshardingNextLeader(newBlock) consensus.getLogger().Info(). Uint64("MsgBlockNum", newBlock.NumberU64()). Msg("[ConsensusMainLoop] Received Proposed New Block!") @@ -404,6 +407,7 @@ func (consensus *Consensus) Start( } // Sleep to wait for the full block time consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") + <-time.After(time.Until(consensus.NextBlockDue)) consensus.StartFinalityCount() @@ -528,6 +532,7 @@ func (consensus *Consensus) getLastMileBlocksAndMsg(bnStart uint64) ([]*types.Bl // preCommitAndPropose commit the current block with 67% commit signatures and start // proposing new block which will wait on the full commit signatures to finish func (consensus *Consensus) preCommitAndPropose(blk *types.Block) error { + //fmt.Println("preCommitAndPropose", utils.GetPort(), blk.NumberU64()) if blk == nil { return errors.New("block to pre-commit is nil") } @@ -645,6 +650,7 @@ func (consensus *Consensus) tryCatchup() error { consensus.getLogger().Error().Err(err).Msg("[TryCatchup] Failed to add block to chain") return err } + //fmt.Println("tryCatchup ", utils.GetPort(), blk.NumberU64()) select { // TODO: Remove this when removing dns sync and stream sync is fully up case consensus.VerifiedNewBlock <- blk: @@ -659,6 +665,8 @@ func (consensus *Consensus) tryCatchup() error { } func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMessage) error { + // this function evaluates for all, leader and validators. + if consensus.Blockchain().CurrentBlock().NumberU64() < blk.NumberU64() { if _, err := consensus.Blockchain().InsertChain([]*types.Block{blk}, !consensus.FBFTLog.IsBlockVerified(blk.Hash())); err != nil { consensus.getLogger().Error().Err(err).Msg("[commitBlock] Failed to add block to chain") @@ -727,10 +735,29 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { // SetupForNewConsensus sets the state for new consensus func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) - consensus.SetCurBlockViewID(committedMsg.ViewID + 1) + curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) // first view id is going to be 2. + prev := consensus.GetLeaderPubKey() + idx := consensus.SetLeaderIndex(func(i int) int { + if curBlockViewID%3 == 0 { + return i + 1 + } + return i + }) + pps := consensus.Decider.Participants() consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] + consensus.LeaderPubKey = &pps[idx%len(pps)] + fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %q\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) consensus.pubKeyLock.Unlock() + if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { + // leader changed + go func() { + fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + consensus.ReadySignal <- SyncProposal + + }() + + } var epoch *big.Int if blk.IsLastBlockInEpoch() { epoch = new(big.Int).Add(blk.Epoch(), common.Big1) diff --git a/consensus/leader.go b/consensus/leader.go index 422508762b..a359c229a4 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -199,9 +199,17 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { + // TODO HERE + //if recvMsg.ViewID == 10 { + // return + //} consensus.mutex.Lock() defer consensus.mutex.Unlock() //// Read - Start + if consensus.ShardID == 0 { + //fmt.Println("onCommit ", recvMsg.BlockNum) + } + if !consensus.isRightBlockNumAndViewID(recvMsg) { return } @@ -333,4 +341,5 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED) } + //fmt.Println("onCommit99: ", utils.GetPort(), recvMsg.BlockNum) } diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 6cf79e549a..0586ace4fe 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -230,12 +230,17 @@ func (s *cIdentities) NthNextHmy(instance shardingconfig.Instance, pubKey *bls.P Msg("[NthNextHmy] pubKey not found") } numNodes := instance.NumHarmonyOperatedNodesPerShard() + //fmt.Println("??idx:", idx, numNodes) // sanity check to avoid out of bound access if numNodes <= 0 || numNodes > len(s.publicKeys) { numNodes = len(s.publicKeys) } idx = (idx + next) % numNodes - return found, &s.publicKeys[idx] + //fmt.Println("-------idx:", idx) + + new := &s.publicKeys[idx] + fmt.Println("NthNextHmy: ", pubKey.Bytes.Hex(), new.Bytes.Hex()) + return found, new } // NthNextHmyExt return the Nth next pubkey of Harmony + allowlist nodes, next can be negative number diff --git a/consensus/validator.go b/consensus/validator.go index a73ac92eb3..013769395c 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -18,6 +18,7 @@ import ( ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { + recvMsg, err := consensus.ParseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). @@ -26,6 +27,9 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Msg("[OnAnnounce] Unparseable leader message") return } + if consensus.ShardID == 0 { + //fmt.Println("onAnnounce called ", recvMsg.BlockNum) + } // NOTE let it handle its own logs if !consensus.onAnnounceSanityChecks(recvMsg) { @@ -184,6 +188,9 @@ func (consensus *Consensus) sendCommitMessages(blockObj *types.Block) { // if onPrepared accepts the prepared message from the leader, then // it will send a COMMIT message for the leader to receive on the network. func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { + if consensus.ShardID == 0 { + //fmt.Println("onPrepared", recvMsg.BlockNum) + } consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -399,6 +406,13 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { consensus.getLogger().Info().Msg("[OnCommitted] Start consensus timer (new block added)") consensus.consensusTimeout[timeoutConsensus].Start() } + + //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum) + if blk != nil { + //consensus.ReshardingNextLeader(blk) + } else { + //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum, "blk is nil") + } } // Collect private keys that are part of the current committee. diff --git a/consensus/view_change.go b/consensus/view_change.go index 6631051dcc..fd9cde6c5f 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync" "time" @@ -161,6 +162,7 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { Uint64("stuckBlockViewID", stuckBlockViewID). Msg("[getNextViewID]") + fmt.Println("end getNextViewID: ", nextViewID, viewChangeDuration) // duration is always the fixed view change duration for synchronous view change return nextViewID, viewChangeDuration } @@ -233,6 +235,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe lastLeaderPubKey, gap) } + fmt.Println("wasfoundNext", consensus.Blockchain.Config().IsAllowlistEpoch(epoch), wasFound, next.Bytes.Hex(), lastLeaderPubKey.Bytes.Hex()) if !wasFound { consensus.getLogger().Warn(). Str("key", consensus.LeaderPubKey.Bytes.Hex()). @@ -254,6 +257,7 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { + fmt.Printf("Message to send leader111: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } @@ -264,6 +268,7 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() + //fmt.Println("startViewChange", nextViewID) consensus.SetViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet @@ -307,7 +312,9 @@ func (consensus *Consensus) startViewChange() { if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { continue } + // Тут уже другой leader msgToSend := consensus.constructViewChangeMessage(&key) + fmt.Println("Message to send leader222: ", consensus.LeaderPubKey.Bytes.Hex()) if err := consensus.msgSender.SendWithRetry( consensus.BlockNum(), msg_pb.MessageType_VIEWCHANGE, @@ -365,6 +372,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.ResetState() } + fmt.Println("[startNewView]", newLeaderPriKey.Pub.Bytes.Hex()) consensus.SetLeaderPubKey(newLeaderPriKey.Pub) return nil @@ -372,6 +380,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri // onViewChange is called when the view change message is received. func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { + //fmt.Printf("[onViewChange] received view change message from %+v\n", recvMsg) consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -394,6 +403,13 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { return } + consensus.getLogger().Debug(). + Err(err). + Interface("SenderPubkeys", recvMsg.SenderPubkeys). + Str("NextLeader", recvMsg.LeaderPubkey.Bytes.Hex()). + Str("myBLSPubKey", consensus.priKey.GetPublicKeys().SerializeToHexStr()). + Msg("[onViewChange] I am the Leader") + if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) { consensus.getLogger().Info(). Int64("have", consensus.Decider.SignersCount(quorum.ViewChange)). @@ -474,6 +490,8 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.mutex.Lock() defer consensus.mutex.Unlock() + fmt.Printf("[onNewView] received new view message from %+v\n", recvMsg) + consensus.getLogger().Info(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -562,6 +580,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // newView message verified success, override my state consensus.SetViewIDs(recvMsg.ViewID) consensus.pubKeyLock.Lock() + fmt.Println("[onNewView1221] new leader key cur:", consensus.LeaderPubKey.Bytes.Hex(), " new: ", senderKey.Bytes.Hex()) consensus.LeaderPubKey = senderKey consensus.pubKeyLock.Unlock() consensus.ResetViewChangeState() diff --git a/hmy/hmy.go b/hmy/hmy.go index 58e8d59edf..8089f04ba8 100644 --- a/hmy/hmy.go +++ b/hmy/hmy.go @@ -18,6 +18,7 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + "github.com/harmony-one/harmony/multibls" commonRPC "github.com/harmony-one/harmony/rpc/common" "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" @@ -96,6 +97,7 @@ type NodeAPI interface { GetStakingTransactionsCount(address, txType string) (uint64, error) GetTraceResultByHash(hash common.Hash) (json.RawMessage, error) IsCurrentlyLeader() bool + GetPublicKeys() multibls.PublicKeys IsOutOfSync(shardID uint32) bool SyncStatus(shardID uint32) (bool, uint64, uint64) SyncPeers() map[string]int diff --git a/node/api.go b/node/api.go index ceda968084..debcb201f6 100644 --- a/node/api.go +++ b/node/api.go @@ -6,6 +6,7 @@ import ( "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy" "github.com/harmony-one/harmony/internal/tikv" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/rosetta" hmy_rpc "github.com/harmony-one/harmony/rpc" rpc_common "github.com/harmony-one/harmony/rpc/common" @@ -18,6 +19,11 @@ func (node *Node) IsCurrentlyLeader() bool { return node.Consensus.IsLeader() } +// GetPublicKeys exposes if node is currently the leader node +func (node *Node) GetPublicKeys() multibls.PublicKeys { + return node.Consensus.GetPrivateKeys().GetPublicKeys() +} + // PeerConnectivity .. func (node *Node) PeerConnectivity() (int, int, int) { return node.host.PeerConnectivity() diff --git a/node/node_newblock.go b/node/node_newblock.go index 03fd69d9dd..24e26ad99a 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -2,6 +2,7 @@ package node import ( "errors" + "fmt" "sort" "strings" "time" @@ -88,6 +89,8 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp newBlock, err := node.ProposeNewBlock(newCommitSigsChan) if err == nil { + fmt.Printf("ProposeNewBlock: #%d :%d @%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), newBlock.Header().ViewID().Int64(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) + if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). Msg("Block with the same number was already proposed, abort.") @@ -145,6 +148,7 @@ func (node *Node) ProposeNewBlock(commitSigs chan []byte) (*types.Block, error) if node.Blockchain().Config().IsStaking(header.Epoch()) { blsPubKeyBytes := leaderKey.Object.GetAddress() coinbase.SetBytes(blsPubKeyBytes[:]) + fmt.Println("coinbase.SetBytes leader: ", leaderKey.Bytes.Hex(), coinbase.Hex()) } emptyAddr := common.Address{} diff --git a/test/configs/local-resharding.txt b/test/configs/local-resharding.txt index 4ce91ece1c..fc3d3e97b4 100644 --- a/test/configs/local-resharding.txt +++ b/test/configs/local-resharding.txt @@ -1,25 +1,12 @@ 127.0.0.1 9000 validator .hmy/65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204.key -127.0.0.1 9002 validator .hmy/40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809.key -127.0.0.1 9004 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key -127.0.0.1 9006 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key -127.0.0.1 9008 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key -127.0.0.1 9010 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key -127.0.0.1 9012 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key -127.0.0.1 9014 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key -127.0.0.1 9016 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key -127.0.0.1 9018 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key -127.0.0.1 9020 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key -127.0.0.1 9022 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key -127.0.0.1 9200 explorer null 0 +127.0.0.1 9002 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key +127.0.0.1 9004 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key +127.0.0.1 9006 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key +127.0.0.1 9008 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key +127.0.0.1 9010 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key +127.0.0.1 9099 explorer null 0 127.0.0.1 9100 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key -127.0.0.1 9102 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key -127.0.0.1 9104 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key -127.0.0.1 9106 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key -127.0.0.1 9108 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key -127.0.0.1 9110 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key -127.0.0.1 9112 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key -127.0.0.1 9114 validator .hmy/f47238daef97d60deedbde5302d05dea5de67608f11f406576e363661f7dcbc4a1385948549b31a6c70f6fde8a391486.key -127.0.0.1 9116 validator .hmy/fc4b9c535ee91f015efff3f32fbb9d32cdd9bfc8a837bb3eee89b8fff653c7af2050a4e147ebe5c7233dc2d5df06ee0a.key -127.0.0.1 9118 validator .hmy/ca86e551ee42adaaa6477322d7db869d3e203c00d7b86c82ebee629ad79cb6d57b8f3db28336778ec2180e56a8e07296.key -127.0.0.1 9300 explorer null 1 \ No newline at end of file +127.0.0.1 9102 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key +127.0.0.1 9104 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key +127.0.0.1 9106 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key \ No newline at end of file diff --git a/test/deploy.sh b/test/deploy.sh index 9ab6943110..fe22de57f0 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -69,7 +69,7 @@ function launch_localnet() { if ${VERBOSE}; then verbosity=5 else - verbosity=3 + verbosity=5 fi base_args=(--log_folder "${log_folder}" --min_peers "${MIN}" --bootnodes "${BN_MA}" "--network_type=$NETWORK" --blspass file:"${ROOT}/.hmy/blspass.txt" "--dns=false" "--verbosity=${verbosity}" "--p2p.security.max-conn-per-ip=100") @@ -80,8 +80,11 @@ function launch_localnet() { while IFS='' read -r line || [[ -n "$line" ]]; do i=$((i + 1)) + + # Read config for i-th node form config file IFS=' ' read -r ip port mode bls_key shard node_config <<<"${line}" + echo "LINE: ${line} ${shard}" args=("${base_args[@]}" --ip "${ip}" --port "${port}" --key "/tmp/${ip}-${port}.key" --db_dir "${ROOT}/db-${ip}-${port}" "--broadcast_invalid_tx=false") if [[ -z "$ip" || -z "$port" ]]; then echo "skip empty node" From f636ad3e1afa46c203455974107d7515da642cd7 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 5 Sep 2022 19:39:49 +0700 Subject: [PATCH 204/420] consensus check is forked --- api/service/explorer/service.go | 6 ++-- consensus/consensus.go | 2 -- consensus/consensus_service.go | 14 -------- consensus/consensus_v2.go | 61 ++++++++++++++++++++++++--------- internal/params/config.go | 11 ++++++ 5 files changed, 58 insertions(+), 36 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index d96c1bc56c..5887bb3a02 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -223,9 +223,9 @@ func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { for i := cur; i > 0; i-- { block := s.blockchain.GetBlockByNumber(i) - w.Write([]byte(fmt.Sprintf("%d ", i))) - w.Write([]byte(fmt.Sprintf("%s ", block.Header().ViewID().String()))) - w.Write([]byte(fmt.Sprintf("%s ", block.Header().Coinbase().Hash().Hex()))) + w.Write([]byte(fmt.Sprintf("#%d ", i))) + w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) + w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) } } diff --git a/consensus/consensus.go b/consensus/consensus.go index 0c51d648c0..e0e096e306 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -73,8 +73,6 @@ type Consensus struct { priKey multibls.PrivateKeys // the publickey of leader LeaderPubKey *bls.PublicKeyWrapper - // index of leader in the list of validators. - LeaderIndex int // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e646504dde..d26ea8ced5 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -474,20 +474,6 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } -// SetLeaderIndex set the leader index. -func (consensus *Consensus) SetLeaderIndex(f func(int) int) (current int) { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() - consensus.LeaderIndex = f(consensus.LeaderIndex) - return consensus.LeaderIndex -} - -func (consensus *Consensus) GetLeaderIndex() int { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() - return consensus.LeaderIndex -} - // SetViewChangingID set the current view change ID func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index bf042c62d3..d92a417836 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -737,25 +737,41 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) // first view id is going to be 2. prev := consensus.GetLeaderPubKey() - idx := consensus.SetLeaderIndex(func(i int) int { - if curBlockViewID%3 == 0 { - return i + 1 + if consensus.Blockchain.Config().IsLeaderRotation(blk.Epoch()) { + epochBlockViewID, err := consensus.getEpochFirstBlockViewID(blk.Epoch()) + if err != nil { + consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", blk.Epoch().Uint64()) + return + } + if epochBlockViewID > curBlockViewID { + consensus.getLogger().Error().Msg("[SetupForNewConsensus] Epoch block viewID is greater than current block viewID") + return } - return i - }) - pps := consensus.Decider.Participants() - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = &pps[idx%len(pps)] - fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %q\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) - consensus.pubKeyLock.Unlock() - if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { - // leader changed - go func() { - fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - consensus.ReadySignal <- SyncProposal - }() + diff := curBlockViewID - epochBlockViewID + + pps := consensus.Decider.Participants() + idx := (int(diff) / 3) % len(pps) + consensus.pubKeyLock.Lock() + fmt.Println("(int(diff)/3)%len(pps) == ", idx) + consensus.LeaderPubKey = &pps[idx] + fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + consensus.pubKeyLock.Unlock() + if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { + // leader changed + go func() { + fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + consensus.ReadySignal <- SyncProposal + + }() + } + } else { + fmt.Printf("SetupForNewConsensus0 :%d future v%d new: %s prev: %s %v\n", utils.GetPort(), curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] + consensus.pubKeyLock.Unlock() + } } var epoch *big.Int @@ -776,6 +792,17 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.ResetState() } +func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, error) { + if epoch.Uint64() == 0 { + return 0, nil + } + epochBlock := consensus.Blockchain.GetBlockByNumber(epoch.Uint64() - 1) + if epochBlock == nil { + return 0, errors.Errorf("block not found for number %d", epoch.Uint64()-1) + } + return epochBlock.Header().ViewID().Uint64() + 1, nil +} + func (consensus *Consensus) postCatchup(initBN uint64) { if initBN < consensus.BlockNum() { consensus.getLogger().Info(). diff --git a/internal/params/config.go b/internal/params/config.go index c11daeafb5..29af19c74f 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -271,6 +271,8 @@ var ( SlotsLimitedEpoch: EpochTBD, // epoch to enable HIP-16 CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, + LeaderRotationEpoch: big.NewInt(1), + TesnetNinetyPercentEpoch: EpochTBD, FeeCollectEpoch: big.NewInt(5), LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 5, @@ -315,6 +317,7 @@ var ( big.NewInt(0), // AllowlistEpoch big.NewInt(1), // LeaderRotationEpoch 64, // LeaderRotationBlocksCount + big.NewInt(1), // LeaderRotationEpoch big.NewInt(0), // FeeCollectEpoch } @@ -711,6 +714,14 @@ func (c *ChainConfig) IsLeaderRotation(epoch *big.Int) bool { return isForked(c.LeaderRotationEpoch, epoch) } +func (c *ChainConfig) IsLeaderRotation(epoch *big.Int) bool { + return isForked(c.LeaderRotationEpoch, epoch) +} + +func (c *ChainConfig) IsTestnetNinetyPercent(epoch *big.Int) bool { + return isForked(c.TesnetNinetyPercentEpoch, epoch) && c == TestnetChainConfig +} + // IsFeeCollectEpoch determines whether Txn Fees will be collected into the community-managed account. func (c *ChainConfig) IsFeeCollectEpoch(epoch *big.Int) bool { return isForked(c.FeeCollectEpoch, epoch) From 2706ea0478dab53f79e47738d5024f65bcb460a6 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sat, 10 Sep 2022 18:02:45 +0700 Subject: [PATCH 205/420] fix --- api/service/explorer/service.go | 2 -- cmd/harmony/main.go | 3 --- consensus/consensus_service.go | 11 ----------- consensus/quorum/quorum.go | 7 +------ 4 files changed, 1 insertion(+), 22 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 5887bb3a02..6aa275332b 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -115,8 +115,6 @@ func (s *Service) Run() *http.Server { s.router = mux.NewRouter() - fmt.Println("++", addr) - // Set up router for addresses. // Fetch addresses request, accepts parameter size: how much addresses to read, // parameter prefix: from which address prefix start diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 2fcfe5f037..4bb70a81b4 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -266,8 +266,6 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { var err error - fmt.Println("OS: ", os.Args) - nodeconfigSetShardSchedule(hc) nodeconfig.SetShardingSchedule(shard.Schedule) nodeconfig.SetVersion(getHarmonyVersion()) @@ -803,7 +801,6 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // Set the consensus ID to be the current block number viewID := currentNode.Blockchain().CurrentBlock().Header().ViewID().Uint64() - fmt.Println("viewID:", viewID) currentConsensus.SetViewIDs(viewID + 1) utils.Logger().Info(). Uint64("viewID", viewID). diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index d26ea8ced5..b010251217 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -213,7 +213,6 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } - fmt.Println("[checkViewID] Set LEADEER PUB KEY ", msg.SenderPubkeys[0].Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() @@ -397,7 +396,6 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { Str("leaderPubKey", leaderPubKey.Bytes.Hex()). Msg("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain") consensus.pubKeyLock.Lock() - fmt.Println("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain", leaderPubKey.Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = leaderPubKey consensus.pubKeyLock.Unlock() } @@ -484,15 +482,6 @@ func (consensus *Consensus) StartFinalityCount() { consensus.finalityCounter.Store(time.Now().UnixNano()) } -//func (consensus *Consensus) ReshardingNextLeader(newblock *types.Block) { -// consensus.pubKeyLock.Lock() -// fmt.Println("nextBlock1 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) -// consensus.LeaderPubKey = consensus.getNextLeaderKey(consensus.GetCurBlockViewID() + 1) -// fmt.Println("nextBlock2 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) -// consensus.pubKeyLock.Unlock() -// -//} - // FinishFinalityCount calculate the current finality func (consensus *Consensus) FinishFinalityCount() { d := time.Now().UnixNano() diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 0586ace4fe..6cf79e549a 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -230,17 +230,12 @@ func (s *cIdentities) NthNextHmy(instance shardingconfig.Instance, pubKey *bls.P Msg("[NthNextHmy] pubKey not found") } numNodes := instance.NumHarmonyOperatedNodesPerShard() - //fmt.Println("??idx:", idx, numNodes) // sanity check to avoid out of bound access if numNodes <= 0 || numNodes > len(s.publicKeys) { numNodes = len(s.publicKeys) } idx = (idx + next) % numNodes - //fmt.Println("-------idx:", idx) - - new := &s.publicKeys[idx] - fmt.Println("NthNextHmy: ", pubKey.Bytes.Hex(), new.Bytes.Hex()) - return found, new + return found, &s.publicKeys[idx] } // NthNextHmyExt return the Nth next pubkey of Harmony + allowlist nodes, next can be negative number From 05b423c3c42d298f2d53ac7a8a4633ff593ce67d Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 15:27:56 +0700 Subject: [PATCH 206/420] Cleanup and fix update pub keys. --- cmd/harmony/main.go | 10 ++++++++++ consensus/consensus.go | 2 +- consensus/consensus_service.go | 4 ++++ consensus/consensus_v2.go | 1 - consensus/leader.go | 11 +++-------- consensus/view_change.go | 11 ----------- 6 files changed, 18 insertions(+), 21 deletions(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 4bb70a81b4..cea2be6c25 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -677,6 +677,16 @@ func createGlobalConfig(hc harmonyconfig.HarmonyConfig) (*nodeconfig.ConfigType, } func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfig.ConfigType, registry *registry.Registry) *node.Node { + // Consensus object. + // TODO: consensus object shouldn't start here + decider := quorum.NewDecider(quorum.SuperMajorityVote, uint32(hc.General.ShardID)) + currentConsensus, err := consensus.New( + myHost, nodeConfig.ShardID, p2p.Peer{}, nodeConfig.ConsensusPriKey, decider) + + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Error :%v \n", err) + os.Exit(1) + } // Parse minPeers from harmonyconfig.HarmonyConfig var minPeers int var aggregateSig bool diff --git a/consensus/consensus.go b/consensus/consensus.go index e0e096e306..ec9d79ce14 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -230,7 +230,7 @@ func (consensus *Consensus) BlockNum() uint64 { // New create a new Consensus record func New( - host p2p.Host, shard uint32, multiBLSPriKey multibls.PrivateKeys, + host p2p.Host, shard uint32, leader p2p.Peer, multiBLSPriKey multibls.PrivateKeys, registry *registry.Registry, Decider quorum.Decider, minPeers int, aggregateSig bool, ) (*Consensus, error) { diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index b010251217..5055684e6c 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -74,6 +74,10 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { + if utils.GetPort() == 9000 { + //utils.Logger().Info().Msg("UpdatePublicKeys") + fmt.Println("UpdatePublicKeys", len(pubKeys), len(allowlist)) + } // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index d92a417836..6b2d6f7025 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -650,7 +650,6 @@ func (consensus *Consensus) tryCatchup() error { consensus.getLogger().Error().Err(err).Msg("[TryCatchup] Failed to add block to chain") return err } - //fmt.Println("tryCatchup ", utils.GetPort(), blk.NumberU64()) select { // TODO: Remove this when removing dns sync and stream sync is fully up case consensus.VerifiedNewBlock <- blk: diff --git a/consensus/leader.go b/consensus/leader.go index a359c229a4..4f72ae6268 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -200,16 +200,12 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { // TODO HERE - //if recvMsg.ViewID == 10 { - // return - //} + if recvMsg.ViewID == 21 { + return + } consensus.mutex.Lock() defer consensus.mutex.Unlock() //// Read - Start - if consensus.ShardID == 0 { - //fmt.Println("onCommit ", recvMsg.BlockNum) - } - if !consensus.isRightBlockNumAndViewID(recvMsg) { return } @@ -341,5 +337,4 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED) } - //fmt.Println("onCommit99: ", utils.GetPort(), recvMsg.BlockNum) } diff --git a/consensus/view_change.go b/consensus/view_change.go index fd9cde6c5f..3bd0c181f2 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync" "time" @@ -162,7 +161,6 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { Uint64("stuckBlockViewID", stuckBlockViewID). Msg("[getNextViewID]") - fmt.Println("end getNextViewID: ", nextViewID, viewChangeDuration) // duration is always the fixed view change duration for synchronous view change return nextViewID, viewChangeDuration } @@ -235,7 +233,6 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe lastLeaderPubKey, gap) } - fmt.Println("wasfoundNext", consensus.Blockchain.Config().IsAllowlistEpoch(epoch), wasFound, next.Bytes.Hex(), lastLeaderPubKey.Bytes.Hex()) if !wasFound { consensus.getLogger().Warn(). Str("key", consensus.LeaderPubKey.Bytes.Hex()). @@ -257,7 +254,6 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - fmt.Printf("Message to send leader111: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } @@ -268,7 +264,6 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() - //fmt.Println("startViewChange", nextViewID) consensus.SetViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet @@ -312,9 +307,7 @@ func (consensus *Consensus) startViewChange() { if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { continue } - // Тут уже другой leader msgToSend := consensus.constructViewChangeMessage(&key) - fmt.Println("Message to send leader222: ", consensus.LeaderPubKey.Bytes.Hex()) if err := consensus.msgSender.SendWithRetry( consensus.BlockNum(), msg_pb.MessageType_VIEWCHANGE, @@ -372,7 +365,6 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.ResetState() } - fmt.Println("[startNewView]", newLeaderPriKey.Pub.Bytes.Hex()) consensus.SetLeaderPubKey(newLeaderPriKey.Pub) return nil @@ -490,8 +482,6 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.mutex.Lock() defer consensus.mutex.Unlock() - fmt.Printf("[onNewView] received new view message from %+v\n", recvMsg) - consensus.getLogger().Info(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -580,7 +570,6 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // newView message verified success, override my state consensus.SetViewIDs(recvMsg.ViewID) consensus.pubKeyLock.Lock() - fmt.Println("[onNewView1221] new leader key cur:", consensus.LeaderPubKey.Bytes.Hex(), " new: ", senderKey.Bytes.Hex()) consensus.LeaderPubKey = senderKey consensus.pubKeyLock.Unlock() consensus.ResetViewChangeState() From cd006791c3eab385aaa8afa03268afe12cdaf98e Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 13 Sep 2022 01:10:13 +0700 Subject: [PATCH 207/420] fix fix fix fix fix --- consensus/consensus_service.go | 5 ----- consensus/leader.go | 4 ---- internal/params/config.go | 9 ++++++--- node/node_newblock.go | 2 +- 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 5055684e6c..1844f22687 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync/atomic" "time" @@ -74,10 +73,6 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { - if utils.GetPort() == 9000 { - //utils.Logger().Info().Msg("UpdatePublicKeys") - fmt.Println("UpdatePublicKeys", len(pubKeys), len(allowlist)) - } // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) diff --git a/consensus/leader.go b/consensus/leader.go index 4f72ae6268..422508762b 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -199,10 +199,6 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { - // TODO HERE - if recvMsg.ViewID == 21 { - return - } consensus.mutex.Lock() defer consensus.mutex.Unlock() //// Read - Start diff --git a/internal/params/config.go b/internal/params/config.go index 29af19c74f..3d8f9762f8 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -271,7 +271,8 @@ var ( SlotsLimitedEpoch: EpochTBD, // epoch to enable HIP-16 CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, - LeaderRotationEpoch: big.NewInt(1), + LeaderRotationEpoch: big.NewInt(4), + LeaderRotationBlocksCount: 5, TesnetNinetyPercentEpoch: EpochTBD, FeeCollectEpoch: big.NewInt(5), LeaderRotationEpoch: EpochTBD, @@ -318,6 +319,7 @@ var ( big.NewInt(1), // LeaderRotationEpoch 64, // LeaderRotationBlocksCount big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } @@ -358,8 +360,9 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount + + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } diff --git a/node/node_newblock.go b/node/node_newblock.go index 24e26ad99a..d59a563ebc 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -89,7 +89,7 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp newBlock, err := node.ProposeNewBlock(newCommitSigsChan) if err == nil { - fmt.Printf("ProposeNewBlock: #%d :%d @%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), newBlock.Header().ViewID().Int64(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) + fmt.Printf("ProposeNewBlock: #%d :%d e:%d, sh:%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), node.Blockchain().CurrentHeader().Epoch().Uint64(), node.Blockchain().ShardID(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). From d2c1ae62cfd8eb3f3990eb60e99fa53b7cdfd835 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 17 Oct 2022 15:08:07 +0700 Subject: [PATCH 208/420] activate epoch --- internal/params/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/params/config.go b/internal/params/config.go index 3d8f9762f8..f1000098a3 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -111,6 +111,7 @@ var ( AllowlistEpoch: big.NewInt(2), LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 64, + TesnetNinetyPercentEpoch: big.NewInt(399), FeeCollectEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. From 2d68c81721b91720a1a6fa90c5cf056a5ac77f5a Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 17 Oct 2022 17:09:04 +0700 Subject: [PATCH 209/420] EpochTBD for leader rotation epoch. --- internal/params/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/params/config.go b/internal/params/config.go index f1000098a3..fc61123aa8 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -110,7 +110,7 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(2), AllowlistEpoch: big.NewInt(2), LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 64, + LeaderRotationBlocksCount: 64, TesnetNinetyPercentEpoch: big.NewInt(399), FeeCollectEpoch: EpochTBD, } From 09d411e6e59766df8bd67554eed462f7c89facd7 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 18 Oct 2022 12:17:17 +0700 Subject: [PATCH 210/420] 295 epoch --- internal/params/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/params/config.go b/internal/params/config.go index fc61123aa8..f1000098a3 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -110,7 +110,7 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(2), AllowlistEpoch: big.NewInt(2), LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 64, + LeaderRotationBlocksCount: 64, TesnetNinetyPercentEpoch: big.NewInt(399), FeeCollectEpoch: EpochTBD, } From d8a6fb4db49ca362f48509f047d59c119cef9d0e Mon Sep 17 00:00:00 2001 From: Konstantin <355847+Frozen@users.noreply.github.com> Date: Fri, 28 Oct 2022 12:52:36 +0300 Subject: [PATCH 211/420] Decider no longer requires public keys as a dependency. (#4289) --- cmd/harmony/main.go | 2 +- consensus/consensus.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index cea2be6c25..d89254ead7 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -681,7 +681,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // TODO: consensus object shouldn't start here decider := quorum.NewDecider(quorum.SuperMajorityVote, uint32(hc.General.ShardID)) currentConsensus, err := consensus.New( - myHost, nodeConfig.ShardID, p2p.Peer{}, nodeConfig.ConsensusPriKey, decider) + myHost, nodeConfig.ShardID, nodeConfig.ConsensusPriKey, decider) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "Error :%v \n", err) diff --git a/consensus/consensus.go b/consensus/consensus.go index ec9d79ce14..e0e096e306 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -230,7 +230,7 @@ func (consensus *Consensus) BlockNum() uint64 { // New create a new Consensus record func New( - host p2p.Host, shard uint32, leader p2p.Peer, multiBLSPriKey multibls.PrivateKeys, + host p2p.Host, shard uint32, multiBLSPriKey multibls.PrivateKeys, registry *registry.Registry, Decider quorum.Decider, minPeers int, aggregateSig bool, ) (*Consensus, error) { From 5de514d45e82f1a95fb48cc1670fa50d19c1208f Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 19 Oct 2022 22:26:13 +0700 Subject: [PATCH 212/420] Consensus doesn't require anymore `Node` as a circular dependency. --- core/blockchain_impl.go | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 8b0683bd65..783a571e89 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -327,19 +327,8 @@ func VerifyBlockCrossLinks(blockchain BlockChain, block *types.Block) error { // ReadCrossLink beacon chain usage. cl, err := blockchain.ReadCrossLink(crossLink.ShardID(), crossLink.BlockNum()) if err == nil && cl != nil { - utils.Logger().Err(errAlreadyExist). - Uint64("beacon-block-number", block.NumberU64()). - Interface("remote", crossLink). - Interface("local", cl). - Msg("[CrossLinkVerification]") - // TODO Add slash for exist same blocknum but different crosslink - return errors.Wrapf( - errAlreadyExist, - "[CrossLinkVerification] shard: %d block: %d on beacon block %d", - crossLink.ShardID(), - crossLink.BlockNum(), - block.NumberU64(), - ) + // Add slash for exist same blocknum but different crosslink + return errAlreadyExist } if err := VerifyCrossLink(blockchain, crossLink); err != nil { return errors.Wrapf(err, "cannot VerifyBlockCrossLinks") From d6f8dae6b250e9f1a062bd98609b02ba41de3043 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Fri, 28 Oct 2022 00:48:12 +0700 Subject: [PATCH 213/420] Proper blockchain initialization. --- cmd/harmony/main.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index d89254ead7..4bb70a81b4 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -677,16 +677,6 @@ func createGlobalConfig(hc harmonyconfig.HarmonyConfig) (*nodeconfig.ConfigType, } func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfig.ConfigType, registry *registry.Registry) *node.Node { - // Consensus object. - // TODO: consensus object shouldn't start here - decider := quorum.NewDecider(quorum.SuperMajorityVote, uint32(hc.General.ShardID)) - currentConsensus, err := consensus.New( - myHost, nodeConfig.ShardID, nodeConfig.ConsensusPriKey, decider) - - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "Error :%v \n", err) - os.Exit(1) - } // Parse minPeers from harmonyconfig.HarmonyConfig var minPeers int var aggregateSig bool From 9ddb48e6330691b8ebb8d5ca50c9384268f9ab28 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 00:33:16 +0800 Subject: [PATCH 214/420] Rwlock consensus. --- api/service/consensus/service.go | 18 ++-- consensus/consensus.go | 17 ++-- consensus/consensus_test.go | 6 +- consensus/consensus_v2.go | 146 +++++++++++++++++-------------- node/node.go | 2 - node/node_newblock.go | 2 +- node/service_setup.go | 2 +- 7 files changed, 99 insertions(+), 94 deletions(-) diff --git a/api/service/consensus/service.go b/api/service/consensus/service.go index d48c660a76..e31ae71bbe 100644 --- a/api/service/consensus/service.go +++ b/api/service/consensus/service.go @@ -3,23 +3,21 @@ package consensus import ( msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus" - "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/utils" ) // Service is the consensus service. type Service struct { - blockChannel chan *types.Block // The channel to receive new blocks from Node - consensus *consensus.Consensus - stopChan chan struct{} - stoppedChan chan struct{} - startChan chan struct{} - messageChan chan *msg_pb.Message + consensus *consensus.Consensus + stopChan chan struct{} + stoppedChan chan struct{} + startChan chan struct{} + messageChan chan *msg_pb.Message } // New returns consensus service. -func New(blockChannel chan *types.Block, consensus *consensus.Consensus, startChan chan struct{}) *Service { - return &Service{blockChannel: blockChannel, consensus: consensus, startChan: startChan} +func New(consensus *consensus.Consensus, startChan chan struct{}) *Service { + return &Service{consensus: consensus, startChan: startChan} } // Start starts consensus service. @@ -27,7 +25,7 @@ func (s *Service) Start() error { utils.Logger().Info().Msg("[consensus/service] Starting consensus service.") s.stopChan = make(chan struct{}) s.stoppedChan = make(chan struct{}) - s.consensus.Start(s.blockChannel, s.stopChan, s.stoppedChan, s.startChan) + s.consensus.Start(s.stopChan, s.stoppedChan, s.startChan) s.consensus.WaitForNewRandomness() return nil } diff --git a/consensus/consensus.go b/consensus/consensus.go index e0e096e306..dbfb39720e 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -85,7 +85,7 @@ type Consensus struct { // IgnoreViewIDCheck determines whether to ignore viewID check IgnoreViewIDCheck *abool.AtomicBool // consensus mutex - mutex sync.Mutex + mutex sync.RWMutex // mutex for verify new block verifyBlockMutex sync.Mutex // ViewChange struct @@ -114,10 +114,6 @@ type Consensus struct { host p2p.Host // MessageSender takes are of sending consensus message and the corresponding retry logic. msgSender *MessageSender - // Used to convey to the consensus main loop that block syncing has finished. - syncReadyChan chan struct{} - // Used to convey to the consensus main loop that node is out of sync - syncNotReadyChan chan struct{} // If true, this consensus will not propose view change. disableViewChange bool // Have a dedicated reader thread pull from this chan, like in node @@ -136,6 +132,9 @@ type Consensus struct { finalityCounter atomic.Value //int64 dHelper *downloadHelper + + // Flag only for initialization state. + start bool } // Blockchain returns the blockchain. @@ -157,12 +156,14 @@ func (consensus *Consensus) VerifyBlock(block *types.Block) error { // BlocksSynchronized lets the main loop know that block synchronization finished // thus the blockchain is likely to be up to date. func (consensus *Consensus) BlocksSynchronized() { - consensus.syncReadyChan <- struct{}{} + consensus.mutex.Lock() + consensus.syncReadyChan() + consensus.mutex.Unlock() } // BlocksNotSynchronized lets the main loop know that block is not synchronized func (consensus *Consensus) BlocksNotSynchronized() { - consensus.syncNotReadyChan <- struct{}{} + consensus.syncNotReadyChan() } // VdfSeedSize returns the number of VRFs for VDF computation @@ -265,8 +266,6 @@ func New( // displayed on explorer as Height right now consensus.SetCurBlockViewID(0) consensus.ShardID = shard - consensus.syncReadyChan = make(chan struct{}) - consensus.syncNotReadyChan = make(chan struct{}) consensus.SlashChan = make(chan slash.Record) consensus.ReadySignal = make(chan ProposalType) consensus.CommitSigChannel = make(chan []byte) diff --git a/consensus/consensus_test.go b/consensus/consensus_test.go index c2d3d729b6..7d74d8815a 100644 --- a/consensus/consensus_test.go +++ b/consensus/consensus_test.go @@ -61,11 +61,7 @@ func TestConsensusInitialization(t *testing.T) { assert.Equal(t, uint64(0), consensus.GetViewChangingID()) assert.Equal(t, uint32(shard.BeaconChainShardID), consensus.ShardID) - assert.IsType(t, make(chan struct{}), consensus.syncReadyChan) - assert.NotNil(t, consensus.syncReadyChan) - - assert.IsType(t, make(chan struct{}), consensus.syncNotReadyChan) - assert.NotNil(t, consensus.syncNotReadyChan) + assert.Equal(t, false, consensus.start) assert.IsType(t, make(chan slash.Record), consensus.SlashChan) assert.NotNil(t, consensus.SlashChan) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 6b2d6f7025..d849898d60 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -292,7 +292,7 @@ func (consensus *Consensus) BlockCommitSigs(blockNum uint64) ([]byte, error) { // Start waits for the next new block and run consensus func (consensus *Consensus) Start( - blockChannel chan *types.Block, stopChan, stoppedChan, startChannel chan struct{}, + stopChan, stoppedChan, startChannel chan struct{}, ) { go func() { toStart := make(chan struct{}, 1) @@ -317,13 +317,13 @@ func (consensus *Consensus) Start( // Set up next block due time. consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) - start := false + consensus.start = false for { select { case <-toStart: - start = true + consensus.start = true case <-ticker.C: - if !start && isInitialLeader { + if !consensus.start && isInitialLeader { continue } for k, v := range consensus.consensusTimeout { @@ -362,68 +362,6 @@ func (consensus *Consensus) Start( } } - // TODO: Refactor this piece of code to consensus/downloader.go after DNS legacy sync is removed - case <-consensus.syncReadyChan: - consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncReadyChan") - consensus.mutex.Lock() - if consensus.BlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { - consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) - consensus.SetViewIDs(consensus.Blockchain().CurrentHeader().ViewID().Uint64() + 1) - mode := consensus.UpdateConsensusInformation() - consensus.current.SetMode(mode) - consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") - consensus.consensusTimeout[timeoutConsensus].Start() - consensus.getLogger().Info().Str("Mode", mode.String()).Msg("Node is IN SYNC") - consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() - } else if consensus.Mode() == Syncing { - // Corner case where sync is triggered before `onCommitted` and there is a race - // for block insertion between consensus and downloader. - mode := consensus.UpdateConsensusInformation() - consensus.SetMode(mode) - consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") - consensus.consensusTimeout[timeoutConsensus].Start() - consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() - } - consensus.mutex.Unlock() - - // TODO: Refactor this piece of code to consensus/downloader.go after DNS legacy sync is removed - case <-consensus.syncNotReadyChan: - consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncNotReadyChan") - consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) - consensus.current.SetMode(Syncing) - consensus.getLogger().Info().Msg("[ConsensusMainLoop] Node is OUT OF SYNC") - consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() - - case newBlock := <-blockChannel: - //consensus.ReshardingNextLeader(newBlock) - consensus.getLogger().Info(). - Uint64("MsgBlockNum", newBlock.NumberU64()). - Msg("[ConsensusMainLoop] Received Proposed New Block!") - - if newBlock.NumberU64() < consensus.BlockNum() { - consensus.getLogger().Warn().Uint64("newBlockNum", newBlock.NumberU64()). - Msg("[ConsensusMainLoop] received old block, abort") - continue - } - // Sleep to wait for the full block time - consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") - - <-time.After(time.Until(consensus.NextBlockDue)) - consensus.StartFinalityCount() - - // Update time due for next block - consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) - - startTime = time.Now() - consensus.msgSender.Reset(newBlock.NumberU64()) - - consensus.getLogger().Info(). - Int("numTxs", len(newBlock.Transactions())). - Int("numStakingTxs", len(newBlock.StakingTransactions())). - Time("startTime", startTime). - Int64("publicKeys", consensus.Decider.ParticipantsCount()). - Msg("[ConsensusMainLoop] STARTING CONSENSUS") - consensus.announce(newBlock) case <-stopChan: consensus.getLogger().Info().Msg("[ConsensusMainLoop] stopChan") return @@ -436,6 +374,82 @@ func (consensus *Consensus) Start( } } +func (consensus *Consensus) syncReadyChan() { + consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncReadyChan") + if consensus.BlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { + consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) + consensus.SetViewIDs(consensus.Blockchain().CurrentHeader().ViewID().Uint64() + 1) + mode := consensus.UpdateConsensusInformation() + consensus.current.SetMode(mode) + consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") + consensus.consensusTimeout[timeoutConsensus].Start() + consensus.getLogger().Info().Str("Mode", mode.String()).Msg("Node is IN SYNC") + consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() + } else if consensus.Mode() == Syncing { + // Corner case where sync is triggered before `onCommitted` and there is a race + // for block insertion between consensus and downloader. + mode := consensus.UpdateConsensusInformation() + consensus.SetMode(mode) + consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") + consensus.consensusTimeout[timeoutConsensus].Start() + consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() + } + +} + +func (consensus *Consensus) syncNotReadyChan() { + consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncNotReadyChan") + consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) + consensus.current.SetMode(Syncing) + consensus.getLogger().Info().Msg("[ConsensusMainLoop] Node is OUT OF SYNC") + consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() +} + +// Close close the consensus. If current is in normal commit phase, wait until the commit +// phase end. +func (consensus *Consensus) Close() error { + if consensus.dHelper != nil { + consensus.dHelper.close() + } + consensus.waitForCommit() + return nil +} + +func (consensus *Consensus) BlockChannel(newBlock *types.Block) { + //consensus.ReshardingNextLeader(newBlock)consensus.getLogger().Info(). + Uint64("MsgBlockNum", newBlock.NumberU64()). + Msg("[ConsensusMainLoop] Received Proposed New Block!") + + if newBlock.NumberU64() < consensus.BlockNum() { + consensus.getLogger().Warn().Uint64("newBlockNum", newBlock.NumberU64()). + Msg("[ConsensusMainLoop] received old block, abort") + return + } + // Sleep to wait for the full block time + consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") + time.AfterFunc(time.Until(consensus.NextBlockDue), func() { + consensus.StartFinalityCount() + + // Update time due for next block + consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) + + startTime = time.Now() + consensus.msgSender.Reset(newBlock.NumberU64()) + + consensus.getLogger().Info(). + Int("numTxs", len(newBlock.Transactions())). + Int("numStakingTxs", len(newBlock.StakingTransactions())). + Time("startTime", startTime). + Int64("publicKeys", consensus.Decider.ParticipantsCount()). + Msg("[ConsensusMainLoop] STARTING CONSENSUS") + consensus.announce(newBlock) + }) + + if consensus.dHelper != nil { + consensus.dHelper.start() + } +} + // Close closes the consensus. If current is in normal commit phase, wait until the commit // phase end. func (consensus *Consensus) Close() error { diff --git a/node/node.go b/node/node.go index 6c2ebcd5fe..a5c1288876 100644 --- a/node/node.go +++ b/node/node.go @@ -100,7 +100,6 @@ type ISync interface { // Node represents a protocol-participating node in the network type Node struct { Consensus *consensus.Consensus // Consensus object containing all Consensus related data (e.g. committee members, signatures, commits) - BlockChannel chan *types.Block // The channel to send newly proposed blocks ConfirmedBlockChannel chan *types.Block // The channel to send confirmed blocks BeaconBlockChannel chan *types.Block // The channel to send beacon blocks for non-beaconchain nodes pendingCXReceipts map[string]*types.CXReceiptsProof // All the receipts received but not yet processed for Consensus @@ -1083,7 +1082,6 @@ func New( } } - node.BlockChannel = make(chan *types.Block) node.ConfirmedBlockChannel = make(chan *types.Block) node.BeaconBlockChannel = make(chan *types.Block) txPoolConfig := core.DefaultTxPoolConfig diff --git a/node/node_newblock.go b/node/node_newblock.go index d59a563ebc..c29fb1b96a 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -107,7 +107,7 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp // Send the new block to Consensus so it can be confirmed. node.proposedBlock[newBlock.NumberU64()] = newBlock delete(node.proposedBlock, newBlock.NumberU64()-10) - node.BlockChannel <- newBlock + node.Consensus.BlockChannel(newBlock) break } else { retryCount++ diff --git a/node/service_setup.go b/node/service_setup.go index 89acaf8fb8..16c554ea8a 100644 --- a/node/service_setup.go +++ b/node/service_setup.go @@ -14,7 +14,7 @@ func (node *Node) RegisterValidatorServices() { // Register consensus service. node.serviceManager.Register( service.Consensus, - consensus.New(node.BlockChannel, node.Consensus, node.startConsensus), + consensus.New(node.Consensus, node.startConsensus), ) // Register new block service. node.serviceManager.Register( From 432ac56a6ab40c97cdc1928e98004b44eefefb03 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 01:11:36 +0800 Subject: [PATCH 215/420] Removed channels. --- api/service/consensus/service.go | 12 +-- consensus/consensus.go | 5 +- consensus/consensus_v2.go | 134 +++++++++++++++---------------- node/node.go | 5 +- node/node_handler.go | 4 +- node/service_setup.go | 2 +- 6 files changed, 76 insertions(+), 86 deletions(-) diff --git a/api/service/consensus/service.go b/api/service/consensus/service.go index e31ae71bbe..d85a072f2e 100644 --- a/api/service/consensus/service.go +++ b/api/service/consensus/service.go @@ -10,22 +10,19 @@ import ( type Service struct { consensus *consensus.Consensus stopChan chan struct{} - stoppedChan chan struct{} - startChan chan struct{} messageChan chan *msg_pb.Message } // New returns consensus service. -func New(consensus *consensus.Consensus, startChan chan struct{}) *Service { - return &Service{consensus: consensus, startChan: startChan} +func New(consensus *consensus.Consensus) *Service { + return &Service{consensus: consensus} } // Start starts consensus service. func (s *Service) Start() error { utils.Logger().Info().Msg("[consensus/service] Starting consensus service.") s.stopChan = make(chan struct{}) - s.stoppedChan = make(chan struct{}) - s.consensus.Start(s.stopChan, s.stoppedChan, s.startChan) + s.consensus.Start(s.stopChan) s.consensus.WaitForNewRandomness() return nil } @@ -33,8 +30,7 @@ func (s *Service) Start() error { // Stop stops consensus service. func (s *Service) Stop() error { utils.Logger().Info().Msg("Stopping consensus service.") - s.stopChan <- struct{}{} - <-s.stoppedChan + close(s.stopChan) utils.Logger().Info().Msg("Consensus service stopped.") return s.consensus.Close() } diff --git a/consensus/consensus.go b/consensus/consensus.go index dbfb39720e..5a131fac5d 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -133,8 +133,9 @@ type Consensus struct { dHelper *downloadHelper - // Flag only for initialization state. - start bool + // Both flags only for initialization state. + start bool + isInitialLeader bool } // Blockchain returns the blockchain. diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index d849898d60..2a133755b7 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -292,81 +292,28 @@ func (consensus *Consensus) BlockCommitSigs(blockNum uint64) ([]byte, error) { // Start waits for the next new block and run consensus func (consensus *Consensus) Start( - stopChan, stoppedChan, startChannel chan struct{}, + stopChan chan struct{}, ) { go func() { - toStart := make(chan struct{}, 1) - isInitialLeader := consensus.IsLeader() - if isInitialLeader { - consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Waiting for consensus start") - // send a signal to indicate it's ready to run consensus - // this signal is consumed by node object to create a new block and in turn trigger a new consensus on it - go func() { - <-startChannel - toStart <- struct{}{} - consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Send ReadySignal") - consensus.ReadySignal <- SyncProposal - }() - } consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Consensus started") - defer close(stoppedChan) - ticker := time.NewTicker(250 * time.Millisecond) - defer ticker.Stop() + go func() { + ticker := time.NewTicker(250 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-stopChan: + return + case <-ticker.C: + consensus.tick() + } + } + }() + consensus.consensusTimeout[timeoutBootstrap].Start() consensus.getLogger().Info().Msg("[ConsensusMainLoop] Start bootstrap timeout (only once)") // Set up next block due time. consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) - consensus.start = false - for { - select { - case <-toStart: - consensus.start = true - case <-ticker.C: - if !consensus.start && isInitialLeader { - continue - } - for k, v := range consensus.consensusTimeout { - // stop timer in listening mode - if consensus.current.Mode() == Listening { - v.Stop() - continue - } - - if consensus.current.Mode() == Syncing { - // never stop bootstrap timer here in syncing mode as it only starts once - // if it is stopped, bootstrap will be stopped and nodes - // can't start view change or join consensus - // the bootstrap timer will be stopped once consensus is reached or view change - // is succeeded - if k != timeoutBootstrap { - consensus.getLogger().Debug(). - Str("k", k.String()). - Str("Mode", consensus.current.Mode().String()). - Msg("[ConsensusMainLoop] consensusTimeout stopped!!!") - v.Stop() - continue - } - } - if !v.CheckExpire() { - continue - } - if k != timeoutViewChange { - consensus.getLogger().Warn().Msg("[ConsensusMainLoop] Ops Consensus Timeout!!!") - consensus.startViewChange() - break - } else { - consensus.getLogger().Warn().Msg("[ConsensusMainLoop] Ops View Change Timeout!!!") - consensus.startViewChange() - break - } - } - - case <-stopChan: - consensus.getLogger().Info().Msg("[ConsensusMainLoop] stopChan") - return - } - } }() if consensus.dHelper != nil { @@ -374,6 +321,15 @@ func (consensus *Consensus) Start( } } +func (consensus *Consensus) StartChannel() { + consensus.isInitialLeader = consensus.IsLeader() + if consensus.isInitialLeader { + consensus.start = true + consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Send ReadySignal") + consensus.ReadySignal <- SyncProposal + } +} + func (consensus *Consensus) syncReadyChan() { consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncReadyChan") if consensus.BlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { @@ -394,7 +350,6 @@ func (consensus *Consensus) syncReadyChan() { consensus.consensusTimeout[timeoutConsensus].Start() consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() } - } func (consensus *Consensus) syncNotReadyChan() { @@ -405,7 +360,48 @@ func (consensus *Consensus) syncNotReadyChan() { consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() } -// Close close the consensus. If current is in normal commit phase, wait until the commit +func (consensus *Consensus) tick() { + if !consensus.start && consensus.isInitialLeader { + return + } + for k, v := range consensus.consensusTimeout { + // stop timer in listening mode + if consensus.current.Mode() == Listening { + v.Stop() + continue + } + + if consensus.current.Mode() == Syncing { + // never stop bootstrap timer here in syncing mode as it only starts once + // if it is stopped, bootstrap will be stopped and nodes + // can't start view change or join consensus + // the bootstrap timer will be stopped once consensus is reached or view change + // is succeeded + if k != timeoutBootstrap { + consensus.getLogger().Debug(). + Str("k", k.String()). + Str("Mode", consensus.current.Mode().String()). + Msg("[ConsensusMainLoop] consensusTimeout stopped!!!") + v.Stop() + continue + } + } + if !v.CheckExpire() { + continue + } + if k != timeoutViewChange { + consensus.getLogger().Warn().Msg("[ConsensusMainLoop] Ops Consensus Timeout!!!") + consensus.startViewChange() + break + } else { + consensus.getLogger().Warn().Msg("[ConsensusMainLoop] Ops View Change Timeout!!!") + consensus.startViewChange() + break + } + } +} + +// Close closes the consensus. If current is in normal commit phase, wait until the commit // phase end. func (consensus *Consensus) Close() error { if consensus.dHelper != nil { diff --git a/node/node.go b/node/node.go index a5c1288876..ba398d6289 100644 --- a/node/node.go +++ b/node/node.go @@ -126,9 +126,7 @@ type Node struct { serviceManager *service.Manager ContractDeployerCurrentNonce uint64 // The nonce of the deployer contract at current block ContractAddresses []common.Address - // Channel to notify consensus service to really start consensus - startConsensus chan struct{} - HarmonyConfig *harmonyconfig.HarmonyConfig + HarmonyConfig *harmonyconfig.HarmonyConfig // node configuration, including group ID, shard ID, etc NodeConfig *nodeconfig.ConfigType // Chain configuration. @@ -1130,7 +1128,6 @@ func New( Msg("Genesis block hash") // Setup initial state of syncing. node.peerRegistrationRecord = map[string]*syncConfig{} - node.startConsensus = make(chan struct{}) // Broadcast double-signers reported by consensus if node.Consensus != nil { go func() { diff --git a/node/node_handler.go b/node/node_handler.go index a34d73e94d..d15b41eb5a 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -434,7 +434,7 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block) error { return nil } -// BootstrapConsensus is the a goroutine to check number of peers and start the consensus +// BootstrapConsensus is a goroutine to check number of peers and start the consensus func (node *Node) BootstrapConsensus() error { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() @@ -464,7 +464,7 @@ func (node *Node) BootstrapConsensus() error { return ctx.Err() case <-enoughMinPeers: go func() { - node.startConsensus <- struct{}{} + node.Consensus.StartChannel() }() return nil } diff --git a/node/service_setup.go b/node/service_setup.go index 16c554ea8a..a2518110c1 100644 --- a/node/service_setup.go +++ b/node/service_setup.go @@ -14,7 +14,7 @@ func (node *Node) RegisterValidatorServices() { // Register consensus service. node.serviceManager.Register( service.Consensus, - consensus.New(node.Consensus, node.startConsensus), + consensus.New(node.Consensus), ) // Register new block service. node.serviceManager.Register( From 95ca3af11c9f5469b7940b29c69ccffc3bdc11e8 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 02:19:35 +0800 Subject: [PATCH 216/420] Removed view change locks. --- consensus/view_change.go | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/consensus/view_change.go b/consensus/view_change.go index 3bd0c181f2..0bfdd92215 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -2,7 +2,6 @@ package consensus import ( "math/big" - "sync" "time" "github.com/harmony-one/harmony/internal/chain" @@ -25,26 +24,21 @@ const MaxViewIDDiff = 249 // State contains current mode and current viewID type State struct { - mode Mode - modeMux sync.RWMutex + mode Mode // current view id in normal mode // it changes per successful consensus blockViewID uint64 - cViewMux sync.RWMutex // view changing id is used during view change mode // it is the next view id viewChangingID uint64 - viewMux sync.RWMutex isBackup bool } // Mode return the current node mode func (pm *State) Mode() Mode { - pm.modeMux.RLock() - defer pm.modeMux.RUnlock() return pm.mode } @@ -54,22 +48,16 @@ func (pm *State) SetMode(s Mode) { s = NormalBackup } - pm.modeMux.Lock() - defer pm.modeMux.Unlock() pm.mode = s } // GetCurBlockViewID return the current view id func (pm *State) GetCurBlockViewID() uint64 { - pm.cViewMux.RLock() - defer pm.cViewMux.RUnlock() return pm.blockViewID } // SetCurBlockViewID sets the current view id func (pm *State) SetCurBlockViewID(viewID uint64) uint64 { - pm.cViewMux.Lock() - defer pm.cViewMux.Unlock() pm.blockViewID = viewID return pm.blockViewID } @@ -77,26 +65,18 @@ func (pm *State) SetCurBlockViewID(viewID uint64) uint64 { // GetViewChangingID return the current view changing id // It is meaningful during view change mode func (pm *State) GetViewChangingID() uint64 { - pm.viewMux.RLock() - defer pm.viewMux.RUnlock() return pm.viewChangingID } // SetViewChangingID set the current view changing id // It is meaningful during view change mode func (pm *State) SetViewChangingID(id uint64) { - pm.viewMux.Lock() - defer pm.viewMux.Unlock() pm.viewChangingID = id } // GetViewChangeDuraion return the duration of the current view change // It increase in the power of difference betweeen view changing ID and current view ID func (pm *State) GetViewChangeDuraion() time.Duration { - pm.viewMux.RLock() - pm.cViewMux.RLock() - defer pm.viewMux.RUnlock() - defer pm.cViewMux.RUnlock() diff := int64(pm.viewChangingID - pm.blockViewID) return time.Duration(diff * diff * int64(viewChangeDuration)) } From 3772fbe06fb0165b507d43548fafb0e1b57066f8 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 02:25:09 +0800 Subject: [PATCH 217/420] Removed timers locks. --- internal/utils/timer.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/internal/utils/timer.go b/internal/utils/timer.go index 3502d68ecc..2e8a77667b 100644 --- a/internal/utils/timer.go +++ b/internal/utils/timer.go @@ -1,7 +1,6 @@ package utils import ( - "sync" "time" ) @@ -20,7 +19,6 @@ type Timeout struct { state TimeoutState d time.Duration start time.Time - mu sync.Mutex } // NewTimeout creates a new timeout class @@ -31,24 +29,18 @@ func NewTimeout(d time.Duration) *Timeout { // Start starts the timeout clock func (timeout *Timeout) Start() { - timeout.mu.Lock() timeout.state = Active timeout.start = time.Now() - timeout.mu.Unlock() } // Stop stops the timeout clock func (timeout *Timeout) Stop() { - timeout.mu.Lock() timeout.state = Inactive timeout.start = time.Now() - timeout.mu.Unlock() } // CheckExpire checks whether the timeout is reached/expired func (timeout *Timeout) CheckExpire() bool { - timeout.mu.Lock() - defer timeout.mu.Unlock() if timeout.state == Active && time.Since(timeout.start) > timeout.d { timeout.state = Expired } @@ -60,23 +52,17 @@ func (timeout *Timeout) CheckExpire() bool { // Duration returns the duration period of timeout func (timeout *Timeout) Duration() time.Duration { - timeout.mu.Lock() - defer timeout.mu.Unlock() return timeout.d } // SetDuration set new duration for the timer func (timeout *Timeout) SetDuration(nd time.Duration) { - timeout.mu.Lock() timeout.d = nd - timeout.mu.Unlock() } // IsActive checks whether timeout clock is active; // A timeout is active means it's not stopped caused by stop // and also not expired with time elapses longer than duration from start func (timeout *Timeout) IsActive() bool { - timeout.mu.Lock() - defer timeout.mu.Unlock() return timeout.state == Active } From a36b8c27cfb9f06acc386cd2daec234e78285085 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 02:28:03 +0800 Subject: [PATCH 218/420] Removed fbft locks. --- consensus/consensus_fbft.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/consensus/consensus_fbft.go b/consensus/consensus_fbft.go index 313abf0615..29c9714d35 100644 --- a/consensus/consensus_fbft.go +++ b/consensus/consensus_fbft.go @@ -1,9 +1,6 @@ package consensus -import "sync" - type LockedFBFTPhase struct { - mu sync.Mutex phase FBFTPhase } @@ -14,14 +11,10 @@ func NewLockedFBFTPhase(initialPhrase FBFTPhase) *LockedFBFTPhase { } func (a *LockedFBFTPhase) Set(phrase FBFTPhase) { - a.mu.Lock() a.phase = phrase - a.mu.Unlock() } func (a *LockedFBFTPhase) Get() FBFTPhase { - a.mu.Lock() - defer a.mu.Unlock() return a.phase } From 240ad811626af451afa68a357e84165fb7cf7349 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 02:38:12 +0800 Subject: [PATCH 219/420] Removed multiSigMutex locks. --- consensus/consensus.go | 1 - consensus/consensus_service.go | 3 +-- consensus/fbft_log.go | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 5a131fac5d..a69bf941bb 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -61,7 +61,6 @@ type Consensus struct { commitBitmap *bls_cosi.Mask multiSigBitmap *bls_cosi.Mask // Bitmap for parsing multisig bitmap from validators - multiSigMutex sync.RWMutex // Registry for services. registry *registry.Registry diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 1844f22687..5534b9bc93 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -138,9 +138,8 @@ func (consensus *Consensus) UpdateBitmaps() { multiSigBitmap, _ := bls_cosi.NewMask(members, nil) consensus.prepareBitmap = prepareBitmap consensus.commitBitmap = commitBitmap - consensus.multiSigMutex.Lock() consensus.multiSigBitmap = multiSigBitmap - consensus.multiSigMutex.Unlock() + } // ResetState resets the state of the consensus diff --git a/consensus/fbft_log.go b/consensus/fbft_log.go index 1a59665729..dead27f1a7 100644 --- a/consensus/fbft_log.go +++ b/consensus/fbft_log.go @@ -360,9 +360,7 @@ func (consensus *Consensus) ParseFBFTMessage(msg *msg_pb.Message) (*FBFTMessage, copy(pbftMsg.SenderPubkeys[0].Bytes[:], consensusMsg.SenderPubkey[:]) } else { // else, it should be a multi-key message where the bitmap is populated - consensus.multiSigMutex.RLock() pubKeys, err := consensus.multiSigBitmap.GetSignedPubKeysFromBitmap(pbftMsg.SenderPubkeyBitmap) - consensus.multiSigMutex.RUnlock() if err != nil { return nil, err } From bfdd7652aad3a556ba5aa84bd7363b7b2f191179 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 02:52:44 +0800 Subject: [PATCH 220/420] Removed leader locks. --- consensus/consensus.go | 5 +--- consensus/consensus_service.go | 7 ------ consensus/consensus_v2.go | 43 +++------------------------------- consensus/view_change.go | 7 +----- 4 files changed, 5 insertions(+), 57 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index a69bf941bb..872eac2a74 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -66,8 +66,7 @@ type Consensus struct { registry *registry.Registry // Minimal number of peers in the shard // If the number of validators is less than minPeers, the consensus won't start - MinPeers int - pubKeyLock sync.Mutex + MinPeers int // private/public keys of current node priKey multibls.PrivateKeys // the publickey of leader @@ -177,8 +176,6 @@ func (consensus *Consensus) GetPublicKeys() multibls.PublicKeys { } func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() return consensus.LeaderPubKey } diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 5534b9bc93..3b46f54b7e 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -73,8 +73,6 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { - // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. - consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) consensus.getLogger().Info().Msg("My Committee updated") for i := range pubKeys { @@ -93,7 +91,6 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi consensus.getLogger().Error(). Msg("[UpdatePublicKeys] Participants is empty") } - consensus.pubKeyLock.Unlock() // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. consensus.UpdateBitmaps() @@ -393,9 +390,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { consensus.getLogger().Info(). Str("leaderPubKey", leaderPubKey.Bytes.Hex()). Msg("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain") - consensus.pubKeyLock.Lock() consensus.LeaderPubKey = leaderPubKey - consensus.pubKeyLock.Unlock() } } @@ -434,9 +429,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { - consensus.pubKeyLock.Lock() obj := consensus.LeaderPubKey.Object - consensus.pubKeyLock.Unlock() for _, key := range consensus.priKey { if key.Pub.Object.IsEqual(obj) { return true diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 2a133755b7..ad3436eaaa 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -744,52 +744,15 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { // SetupForNewConsensus sets the state for new consensus func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) - curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) // first view id is going to be 2. - prev := consensus.GetLeaderPubKey() - if consensus.Blockchain.Config().IsLeaderRotation(blk.Epoch()) { - epochBlockViewID, err := consensus.getEpochFirstBlockViewID(blk.Epoch()) - if err != nil { - consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", blk.Epoch().Uint64()) - return - } - if epochBlockViewID > curBlockViewID { - consensus.getLogger().Error().Msg("[SetupForNewConsensus] Epoch block viewID is greater than current block viewID") - return - } - - diff := curBlockViewID - epochBlockViewID - - pps := consensus.Decider.Participants() - idx := (int(diff) / 3) % len(pps) - consensus.pubKeyLock.Lock() - fmt.Println("(int(diff)/3)%len(pps) == ", idx) - consensus.LeaderPubKey = &pps[idx] - fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) - consensus.pubKeyLock.Unlock() - if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { - // leader changed - go func() { - fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - consensus.ReadySignal <- SyncProposal - - }() - } - } else { - fmt.Printf("SetupForNewConsensus0 :%d future v%d new: %s prev: %s %v\n", utils.GetPort(), curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] - consensus.pubKeyLock.Unlock() - } - - } + consensus.SetCurBlockViewID(committedMsg.ViewID + 1) + consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] var epoch *big.Int if blk.IsLastBlockInEpoch() { epoch = new(big.Int).Add(blk.Epoch(), common.Big1) } else { epoch = blk.Epoch() } - if consensus.Blockchain().Config().IsLeaderRotation(epoch) { + if consensus.Blockchain.Config().IsLeaderRotation(epoch) { consensus.rotateLeader(epoch) } diff --git a/consensus/view_change.go b/consensus/view_change.go index 0bfdd92215..1267684d66 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -251,10 +251,7 @@ func (consensus *Consensus) startViewChange() { // aganist the consensus.LeaderPubKey variable. // Ideally, we shall use another variable to keep track of the // leader pubkey in viewchange mode - consensus.pubKeyLock.Lock() - lpk := consensus.getNextLeaderKey(nextViewID) - consensus.LeaderPubKey = lpk - consensus.pubKeyLock.Unlock() + consensus.LeaderPubKey = consensus.getNextLeaderKey(nextViewID) consensus.getLogger().Warn(). Uint64("nextViewID", nextViewID). @@ -549,9 +546,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // newView message verified success, override my state consensus.SetViewIDs(recvMsg.ViewID) - consensus.pubKeyLock.Lock() consensus.LeaderPubKey = senderKey - consensus.pubKeyLock.Unlock() consensus.ResetViewChangeState() consensus.msgSender.StopRetry(msg_pb.MessageType_VIEWCHANGE) From 8d4876e12cde5c015ae9dbb39e7cba8d55a0b775 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 16:51:24 +0800 Subject: [PATCH 221/420] Removed additional locks and isViewChange. --- consensus/checks.go | 4 +-- consensus/consensus_service.go | 2 +- consensus/consensus_v2.go | 10 +++++-- consensus/fbft_log.go | 51 +--------------------------------- consensus/validator.go | 4 +-- consensus/view_change.go | 8 ++---- 6 files changed, 17 insertions(+), 62 deletions(-) diff --git a/consensus/checks.go b/consensus/checks.go index c44a58da39..9e095a45c4 100644 --- a/consensus/checks.go +++ b/consensus/checks.go @@ -86,7 +86,7 @@ func (consensus *Consensus) onAnnounceSanityChecks(recvMsg *FBFTMessage) bool { Str("recvMsg", recvMsg.String()). Str("LeaderKey", consensus.LeaderPubKey.Bytes.Hex()). Msg("[OnAnnounce] Leader is malicious") - if consensus.IsViewChangingMode() { + if consensus.isViewChangingMode() { consensus.getLogger().Debug().Msg( "[OnAnnounce] Already in ViewChanging mode, conflicing announce, doing noop", ) @@ -161,7 +161,7 @@ func (consensus *Consensus) onViewChangeSanityCheck(recvMsg *FBFTMessage) bool { Msg("[onViewChangeSanityCheck] MsgBlockNum is different from my BlockNumber") return false } - if consensus.IsViewChangingMode() && + if consensus.isViewChangingMode() && consensus.GetCurBlockViewID() > recvMsg.ViewID { consensus.getLogger().Debug().Uint64("curBlockViewID", consensus.GetCurBlockViewID()). Uint64("msgViewID", recvMsg.ViewID). diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 3b46f54b7e..ee0e6026c6 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -97,7 +97,7 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi consensus.ResetState() // do not reset view change state if it is in view changing mode - if !consensus.IsViewChangingMode() { + if !consensus.isViewChangingMode() { consensus.ResetViewChangeState() } return consensus.Decider.ParticipantsCount() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index ad3436eaaa..710c4e72b8 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -47,6 +47,12 @@ const ( // IsViewChangingMode return true if curernt mode is viewchanging func (consensus *Consensus) IsViewChangingMode() bool { + //consensus.mutex.RLock() + //defer consensus.mutex.RUnlock() + return consensus.isViewChangingMode() +} + +func (consensus *Consensus) isViewChangingMode() bool { return consensus.current.Mode() == ViewChanging } @@ -56,7 +62,7 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb // in order to avoid possible trap forever but drop PREPARE and COMMIT // which are message types specifically for a node acting as leader // so we just ignore those messages - if consensus.IsViewChangingMode() && + if consensus.isViewChangingMode() && (msg.Type == msg_pb.MessageType_PREPARE || msg.Type == msg_pb.MessageType_COMMIT) { return nil @@ -784,7 +790,7 @@ func (consensus *Consensus) postCatchup(initBN uint64) { consensus.switchPhase("TryCatchup", FBFTAnnounce) } // catch up and skip from view change trap - if initBN < consensus.BlockNum() && consensus.IsViewChangingMode() { + if initBN < consensus.BlockNum() && consensus.isViewChangingMode() { consensus.current.SetMode(Normal) consensus.consensusTimeout[timeoutViewChange].Stop() } diff --git a/consensus/fbft_log.go b/consensus/fbft_log.go index dead27f1a7..4743085ab6 100644 --- a/consensus/fbft_log.go +++ b/consensus/fbft_log.go @@ -3,7 +3,6 @@ package consensus import ( "encoding/binary" "fmt" - "sync" "github.com/ethereum/go-ethereum/common" bls_core "github.com/harmony-one/bls/ffi/go/bls" @@ -102,10 +101,7 @@ func (m *FBFTMessage) id() fbftMsgID { type FBFTLog struct { blocks map[common.Hash]*types.Block // store blocks received in FBFT verifiedBlocks map[common.Hash]struct{} // store block hashes for blocks that has already been verified - blockLock sync.RWMutex - - messages map[fbftMsgID]*FBFTMessage // store messages received in FBFT - msgLock sync.RWMutex + messages map[fbftMsgID]*FBFTMessage // store messages received in FBFT } // NewFBFTLog returns new instance of FBFTLog @@ -120,42 +116,27 @@ func NewFBFTLog() *FBFTLog { // AddBlock add a new block into the log func (log *FBFTLog) AddBlock(block *types.Block) { - log.blockLock.Lock() - defer log.blockLock.Unlock() - log.blocks[block.Hash()] = block } // MarkBlockVerified marks the block as verified func (log *FBFTLog) MarkBlockVerified(block *types.Block) { - log.blockLock.Lock() - defer log.blockLock.Unlock() - log.verifiedBlocks[block.Hash()] = struct{}{} } // IsBlockVerified checks whether the block is verified func (log *FBFTLog) IsBlockVerified(hash common.Hash) bool { - log.blockLock.RLock() - defer log.blockLock.RUnlock() - _, exist := log.verifiedBlocks[hash] return exist } // GetBlockByHash returns the block matches the given block hash func (log *FBFTLog) GetBlockByHash(hash common.Hash) *types.Block { - log.blockLock.RLock() - defer log.blockLock.RUnlock() - return log.blocks[hash] } // GetBlocksByNumber returns the blocks match the given block number func (log *FBFTLog) GetBlocksByNumber(number uint64) []*types.Block { - log.blockLock.RLock() - defer log.blockLock.RUnlock() - var blocks []*types.Block for _, block := range log.blocks { if block.NumberU64() == number { @@ -167,9 +148,6 @@ func (log *FBFTLog) GetBlocksByNumber(number uint64) []*types.Block { // DeleteBlocksLessThan deletes blocks less than given block number func (log *FBFTLog) DeleteBlocksLessThan(number uint64) { - log.blockLock.Lock() - defer log.blockLock.Unlock() - for h, block := range log.blocks { if block.NumberU64() < number { delete(log.blocks, h) @@ -180,9 +158,6 @@ func (log *FBFTLog) DeleteBlocksLessThan(number uint64) { // DeleteBlockByNumber deletes block of specific number func (log *FBFTLog) DeleteBlockByNumber(number uint64) { - log.blockLock.Lock() - defer log.blockLock.Unlock() - for h, block := range log.blocks { if block.NumberU64() == number { delete(log.blocks, h) @@ -193,9 +168,6 @@ func (log *FBFTLog) DeleteBlockByNumber(number uint64) { // DeleteMessagesLessThan deletes messages less than given block number func (log *FBFTLog) DeleteMessagesLessThan(number uint64) { - log.msgLock.Lock() - defer log.msgLock.Unlock() - for h, msg := range log.messages { if msg.BlockNum < number { delete(log.messages, h) @@ -205,9 +177,6 @@ func (log *FBFTLog) DeleteMessagesLessThan(number uint64) { // AddVerifiedMessage adds a signature verified pbft message into the log func (log *FBFTLog) AddVerifiedMessage(msg *FBFTMessage) { - log.msgLock.Lock() - defer log.msgLock.Unlock() - msg.Verified = true log.messages[msg.id()] = msg @@ -215,9 +184,6 @@ func (log *FBFTLog) AddVerifiedMessage(msg *FBFTMessage) { // AddNotVerifiedMessage adds a not signature verified pbft message into the log func (log *FBFTLog) AddNotVerifiedMessage(msg *FBFTMessage) { - log.msgLock.Lock() - defer log.msgLock.Unlock() - msg.Verified = false log.messages[msg.id()] = msg @@ -225,9 +191,6 @@ func (log *FBFTLog) AddNotVerifiedMessage(msg *FBFTMessage) { // GetNotVerifiedCommittedMessages returns not verified committed pbft messages with matching blockNum, viewID and blockHash func (log *FBFTLog) GetNotVerifiedCommittedMessages(blockNum uint64, viewID uint64, blockHash common.Hash) []*FBFTMessage { - log.msgLock.RLock() - defer log.msgLock.RUnlock() - var found []*FBFTMessage for _, msg := range log.messages { if msg.MessageType == msg_pb.MessageType_COMMITTED && msg.BlockNum == blockNum && msg.ViewID == viewID && msg.BlockHash == blockHash && !msg.Verified { @@ -239,9 +202,6 @@ func (log *FBFTLog) GetNotVerifiedCommittedMessages(blockNum uint64, viewID uint // GetMessagesByTypeSeqViewHash returns pbft messages with matching type, blockNum, viewID and blockHash func (log *FBFTLog) GetMessagesByTypeSeqViewHash(typ msg_pb.MessageType, blockNum uint64, viewID uint64, blockHash common.Hash) []*FBFTMessage { - log.msgLock.RLock() - defer log.msgLock.RUnlock() - var found []*FBFTMessage for _, msg := range log.messages { if msg.MessageType == typ && msg.BlockNum == blockNum && msg.ViewID == viewID && msg.BlockHash == blockHash && msg.Verified { @@ -253,9 +213,6 @@ func (log *FBFTLog) GetMessagesByTypeSeqViewHash(typ msg_pb.MessageType, blockNu // GetMessagesByTypeSeq returns pbft messages with matching type, blockNum func (log *FBFTLog) GetMessagesByTypeSeq(typ msg_pb.MessageType, blockNum uint64) []*FBFTMessage { - log.msgLock.RLock() - defer log.msgLock.RUnlock() - var found []*FBFTMessage for _, msg := range log.messages { if msg.MessageType == typ && msg.BlockNum == blockNum && msg.Verified { @@ -267,9 +224,6 @@ func (log *FBFTLog) GetMessagesByTypeSeq(typ msg_pb.MessageType, blockNum uint64 // GetMessagesByTypeSeqHash returns pbft messages with matching type, blockNum func (log *FBFTLog) GetMessagesByTypeSeqHash(typ msg_pb.MessageType, blockNum uint64, blockHash common.Hash) []*FBFTMessage { - log.msgLock.RLock() - defer log.msgLock.RUnlock() - var found []*FBFTMessage for _, msg := range log.messages { if msg.MessageType == typ && msg.BlockNum == blockNum && msg.BlockHash == blockHash && msg.Verified { @@ -305,9 +259,6 @@ func (log *FBFTLog) HasMatchingViewPrepared(blockNum uint64, viewID uint64, bloc // GetMessagesByTypeSeqView returns pbft messages with matching type, blockNum and viewID func (log *FBFTLog) GetMessagesByTypeSeqView(typ msg_pb.MessageType, blockNum uint64, viewID uint64) []*FBFTMessage { - log.msgLock.RLock() - defer log.msgLock.RUnlock() - var found []*FBFTMessage for _, msg := range log.messages { if msg.MessageType != typ || msg.BlockNum != blockNum || msg.ViewID != viewID && msg.Verified { diff --git a/consensus/validator.go b/consensus/validator.go index 013769395c..3e958fef26 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -47,7 +47,7 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { consensus.blockHash = recvMsg.BlockHash // we have already added message and block, skip check viewID // and send prepare message if is in ViewChanging mode - if consensus.IsViewChangingMode() { + if consensus.isViewChangingMode() { consensus.getLogger().Debug(). Msg("[OnAnnounce] Still in ViewChanging Mode, Exiting !!") return @@ -392,7 +392,7 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { return } - if consensus.IsViewChangingMode() { + if consensus.isViewChangingMode() { consensus.getLogger().Info().Msg("[OnCommitted] Still in ViewChanging mode, Exiting!!") return } diff --git a/consensus/view_change.go b/consensus/view_change.go index 1267684d66..aff25c2d15 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -237,8 +237,6 @@ func (consensus *Consensus) startViewChange() { if consensus.disableViewChange || consensus.IsBackup() { return } - consensus.mutex.Lock() - defer consensus.mutex.Unlock() consensus.consensusTimeout[timeoutConsensus].Stop() consensus.consensusTimeout[timeoutBootstrap].Stop() @@ -300,7 +298,7 @@ func (consensus *Consensus) startViewChange() { // startNewView stops the current view change func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.PrivateKeyWrapper, reset bool) error { - if !consensus.IsViewChangingMode() { + if !consensus.isViewChangingMode() { return errors.New("not in view changing mode anymore") } @@ -421,7 +419,7 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { } // received enough view change messages, change state to normal consensus - if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) && consensus.IsViewChangingMode() { + if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) && consensus.isViewChangingMode() { // no previous prepared message, go straight to normal mode // and start proposing new block if consensus.vc.IsM1PayloadEmpty() { @@ -537,7 +535,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { } } - if !consensus.IsViewChangingMode() { + if !consensus.isViewChangingMode() { consensus.getLogger().Info().Msg("Not in ViewChanging Mode.") return } From 250695604fcc9c303de8e7f47d1b7ebf7e115579 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 18:37:20 +0800 Subject: [PATCH 222/420] Added locks detected by race. --- consensus/consensus_v2.go | 5 ++++- scripts/go_executable_build.sh | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 710c4e72b8..4575e197ef 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -310,16 +310,19 @@ func (consensus *Consensus) Start( case <-stopChan: return case <-ticker.C: + consensus.mutex.Lock() consensus.tick() + consensus.mutex.Unlock() } } }() + consensus.mutex.Lock() consensus.consensusTimeout[timeoutBootstrap].Start() consensus.getLogger().Info().Msg("[ConsensusMainLoop] Start bootstrap timeout (only once)") - // Set up next block due time. consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) + consensus.mutex.Unlock() }() if consensus.dHelper != nil { diff --git a/scripts/go_executable_build.sh b/scripts/go_executable_build.sh index 2c1cc99f0e..1aae7c9df7 100755 --- a/scripts/go_executable_build.sh +++ b/scripts/go_executable_build.sh @@ -120,6 +120,7 @@ function build_only if [ "$STATIC" == "true" ]; then env GOOS=$GOOS GOARCH=$GOARCH go build $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY} -w -extldflags \"-static -lm\"" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} else +# env GOOS=$GOOS GOARCH=$GOARCH go build -race $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY}" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} env GOOS=$GOOS GOARCH=$GOARCH go build $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY}" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} fi fi From 8be3607bf0115a5743133df921c0998536f7ce1b Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 18:37:42 +0800 Subject: [PATCH 223/420] Added locks detected by race. --- scripts/go_executable_build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/go_executable_build.sh b/scripts/go_executable_build.sh index 1aae7c9df7..2c1cc99f0e 100755 --- a/scripts/go_executable_build.sh +++ b/scripts/go_executable_build.sh @@ -120,7 +120,6 @@ function build_only if [ "$STATIC" == "true" ]; then env GOOS=$GOOS GOARCH=$GOARCH go build $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY} -w -extldflags \"-static -lm\"" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} else -# env GOOS=$GOOS GOARCH=$GOARCH go build -race $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY}" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} env GOOS=$GOOS GOARCH=$GOARCH go build $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY}" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} fi fi From 696f1a081c6b6805325a6a4a862714ca378e6b78 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 19:38:31 +0800 Subject: [PATCH 224/420] Locks for start. --- consensus/consensus_v2.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 4575e197ef..4db16a3725 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -331,12 +331,16 @@ func (consensus *Consensus) Start( } func (consensus *Consensus) StartChannel() { + consensus.mutex.Lock() consensus.isInitialLeader = consensus.IsLeader() if consensus.isInitialLeader { consensus.start = true consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Send ReadySignal") + consensus.mutex.Unlock() consensus.ReadySignal <- SyncProposal + return } + consensus.mutex.Unlock() } func (consensus *Consensus) syncReadyChan() { From 51814a5bd99a1fc6117f99df3005c6ff31184882 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 20:03:09 +0800 Subject: [PATCH 225/420] Removed additional logs. --- consensus/leader.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/consensus/leader.go b/consensus/leader.go index 422508762b..6b176d27ae 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -26,12 +26,8 @@ func (consensus *Consensus) announce(block *types.Block) { return } - //// Lock Write - Start - consensus.mutex.Lock() copy(consensus.blockHash[:], blockHash[:]) consensus.block = encodedBlock // Must set block bytes before consensus.construct() - consensus.mutex.Unlock() - //// Lock Write - End key, err := consensus.GetConsensusLeaderPrivateKey() if err != nil { From 44a9f83416216b18e02efdd2840f114d639ddc9e Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 20:35:59 +0800 Subject: [PATCH 226/420] Removed additional locks. --- consensus/consensus_service.go | 1 - consensus/consensus_v2.go | 2 -- 2 files changed, 3 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index ee0e6026c6..9a171ea1d0 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -493,7 +493,6 @@ func (consensus *Consensus) switchPhase(subject string, desired FBFTPhase) { Str("switchPhase:", subject) consensus.phase.Set(desired) - return } var ( diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 4db16a3725..876613d819 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -47,8 +47,6 @@ const ( // IsViewChangingMode return true if curernt mode is viewchanging func (consensus *Consensus) IsViewChangingMode() bool { - //consensus.mutex.RLock() - //defer consensus.mutex.RUnlock() return consensus.isViewChangingMode() } From 5deedd594b3e4f0c13c0885cd0fb688634e30bf5 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 3 Nov 2022 01:58:08 +0800 Subject: [PATCH 227/420] Removed additional locks. --- consensus/consensus.go | 6 ++++-- consensus/consensus_service.go | 2 +- consensus/validator.go | 10 ---------- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 872eac2a74..b24a303ccf 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -84,8 +84,6 @@ type Consensus struct { IgnoreViewIDCheck *abool.AtomicBool // consensus mutex mutex sync.RWMutex - // mutex for verify new block - verifyBlockMutex sync.Mutex // ViewChange struct vc *viewChange // Signal channel for proposing a new block and start new consensus @@ -172,6 +170,10 @@ func (consensus *Consensus) VdfSeedSize() int { // GetPublicKeys returns the public keys func (consensus *Consensus) GetPublicKeys() multibls.PublicKeys { + return consensus.getPublicKeys() +} + +func (consensus *Consensus) getPublicKeys() multibls.PublicKeys { return consensus.priKey.GetPublicKeys() } diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 9a171ea1d0..aec2e2b94b 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -396,7 +396,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { for _, key := range pubKeys { // in committee - myPubKeys := consensus.GetPublicKeys() + myPubKeys := consensus.getPublicKeys() if myPubKeys.Contains(key.Object) { if hasError { consensus.getLogger().Error(). diff --git a/consensus/validator.go b/consensus/validator.go index 3e958fef26..ffdcedb354 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -79,10 +79,6 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { } func (consensus *Consensus) validateNewBlock(recvMsg *FBFTMessage) (*types.Block, error) { - // Lock to prevent race condition between announce and prepare - consensus.verifyBlockMutex.Lock() - defer consensus.verifyBlockMutex.Unlock() - if consensus.FBFTLog.IsBlockVerified(recvMsg.BlockHash) { var blockObj *types.Block @@ -188,12 +184,6 @@ func (consensus *Consensus) sendCommitMessages(blockObj *types.Block) { // if onPrepared accepts the prepared message from the leader, then // it will send a COMMIT message for the leader to receive on the network. func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { - if consensus.ShardID == 0 { - //fmt.Println("onPrepared", recvMsg.BlockNum) - } - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - consensus.getLogger().Info(). Uint64("MsgBlockNum", recvMsg.BlockNum). Uint64("MsgViewID", recvMsg.ViewID). From 0f353516aedfc7b76d490bee17853cb5b0793915 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sun, 6 Nov 2022 22:47:14 +0700 Subject: [PATCH 228/420] Make func private. --- consensus/consensus_service.go | 6 +++--- consensus/consensus_v2.go | 2 +- consensus/view_change.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index aec2e2b94b..bf420177cc 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -94,7 +94,7 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. consensus.UpdateBitmaps() - consensus.ResetState() + consensus.resetState() // do not reset view change state if it is in view changing mode if !consensus.isViewChangingMode() { @@ -140,7 +140,7 @@ func (consensus *Consensus) UpdateBitmaps() { } // ResetState resets the state of the consensus -func (consensus *Consensus) ResetState() { +func (consensus *Consensus) resetState() { consensus.switchPhase("ResetState", FBFTAnnounce) consensus.blockHash = [32]byte{} @@ -520,7 +520,7 @@ func (consensus *Consensus) selfCommit(payload []byte) error { } // Have to keep the block hash so the leader can finish the commit phase of prepared block - consensus.ResetState() + consensus.resetState() copy(consensus.blockHash[:], blockHash[:]) consensus.switchPhase("selfCommit", FBFTCommit) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 876613d819..763214c035 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -772,7 +772,7 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.SetMode(consensus.UpdateConsensusInformation()) } consensus.FBFTLog.PruneCacheBeforeBlock(blk.NumberU64()) - consensus.ResetState() + consensus.resetState() } func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, error) { diff --git a/consensus/view_change.go b/consensus/view_change.go index aff25c2d15..95ccbb97aa 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -338,7 +338,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri // TODO: consider make ResetState unified and only called in one place like finalizeCommit() if reset { - consensus.ResetState() + consensus.resetState() } consensus.SetLeaderPubKey(newLeaderPriKey.Pub) @@ -554,7 +554,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.sendCommitMessages(preparedBlock) consensus.switchPhase("onNewView", FBFTCommit) } else { - consensus.ResetState() + consensus.resetState() consensus.getLogger().Info().Msg("onNewView === announce") } consensus.getLogger().Info(). From a8d3e1ea95035a1b4b05fa771a42725a88be5eac Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sun, 6 Nov 2022 23:42:26 +0700 Subject: [PATCH 229/420] Make VerifyBlock private. --- consensus/consensus.go | 4 ++-- consensus/consensus_v2.go | 2 +- consensus/validator.go | 9 ++++++--- consensus/view_change_msg.go | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index b24a303ccf..887b4744e9 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -140,7 +140,7 @@ func (consensus *Consensus) Blockchain() core.BlockChain { } // VerifyBlock is a function used to verify the block and keep trace of verified blocks. -func (consensus *Consensus) VerifyBlock(block *types.Block) error { +func (consensus *Consensus) verifyBlock(block *types.Block) error { if !consensus.FBFTLog.IsBlockVerified(block.Hash()) { if err := consensus.BlockVerifier(block); err != nil { return errors.Errorf("Block verification failed: %s", err) @@ -217,7 +217,7 @@ func (consensus *Consensus) GetConsensusLeaderPrivateKey() (*bls.PrivateKeyWrapp // SetBlockVerifier sets the block verifier func (consensus *Consensus) SetBlockVerifier(verifier VerifyBlockFunc) { consensus.BlockVerifier = verifier - consensus.vc.SetVerifyBlock(consensus.VerifyBlock) + consensus.vc.SetVerifyBlock(consensus.verifyBlock) } func (consensus *Consensus) IsBackup() bool { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 763214c035..18feeae286 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -662,7 +662,7 @@ func (consensus *Consensus) tryCatchup() error { } blk.SetCurrentCommitSig(msg.Payload) - if err := consensus.VerifyBlock(blk); err != nil { + if err := consensus.verifyBlock(blk); err != nil { consensus.getLogger().Err(err).Msg("[TryCatchup] failed block verifier") return err } diff --git a/consensus/validator.go b/consensus/validator.go index ffdcedb354..b33ac6d688 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -42,8 +42,6 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Uint64("MsgBlockNum", recvMsg.BlockNum). Msg("[OnAnnounce] Announce message Added") consensus.FBFTLog.AddVerifiedMessage(recvMsg) - consensus.mutex.Lock() - defer consensus.mutex.Unlock() consensus.blockHash = recvMsg.BlockHash // we have already added message and block, skip check viewID // and send prepare message if is in ViewChanging mode @@ -78,6 +76,11 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { } } +func (consensus *Consensus) ValidateNewBlock(recvMsg *FBFTMessage) (*types.Block, error) { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + return consensus.validateNewBlock(recvMsg) +} func (consensus *Consensus) validateNewBlock(recvMsg *FBFTMessage) (*types.Block, error) { if consensus.FBFTLog.IsBlockVerified(recvMsg.BlockHash) { var blockObj *types.Block @@ -132,7 +135,7 @@ func (consensus *Consensus) validateNewBlock(recvMsg *FBFTMessage) (*types.Block return nil, errors.New("nil block verifier") } - if err := consensus.VerifyBlock(&blockObj); err != nil { + if err := consensus.verifyBlock(&blockObj); err != nil { consensus.getLogger().Error().Err(err).Msg("[validateNewBlock] Block verification failed") return nil, errors.New("Block verification failed") } diff --git a/consensus/view_change_msg.go b/consensus/view_change_msg.go index be19741055..2433abdcbe 100644 --- a/consensus/view_change_msg.go +++ b/consensus/view_change_msg.go @@ -45,7 +45,7 @@ func (consensus *Consensus) constructViewChangeMessage(priKey *bls.PrivateKeyWra Interface("preparedMsg", preparedMsg). Msg("[constructViewChangeMessage] found prepared msg") if block != nil { - if err := consensus.VerifyBlock(block); err == nil { + if err := consensus.verifyBlock(block); err == nil { tmpEncoded, err := rlp.EncodeToBytes(block) if err != nil { consensus.getLogger().Err(err).Msg("[constructViewChangeMessage] Failed encoding block") From f5e2b812c5f325507973a16a87f5bf25f8a2171e Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sun, 6 Nov 2022 23:55:22 +0700 Subject: [PATCH 230/420] Make IsLeader private. --- consensus/consensus_service.go | 11 ++++++++++- consensus/consensus_v2.go | 10 +++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index bf420177cc..fdbcbc0457 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -408,7 +408,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // If the leader changed and I myself become the leader if (oldLeader != nil && consensus.LeaderPubKey != nil && - !consensus.LeaderPubKey.Object.IsEqual(oldLeader.Object)) && consensus.IsLeader() { + !consensus.LeaderPubKey.Object.IsEqual(oldLeader.Object)) && consensus.isLeader() { go func() { consensus.getLogger().Info(). Str("myKey", myPubKeys.SerializeToHexStr()). @@ -429,6 +429,15 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() + + return consensus.isLeader() +} + +// IsLeader check if the node is a leader or not by comparing the public key of +// the node with the leader public key +func (consensus *Consensus) isLeader() bool { obj := consensus.LeaderPubKey.Object for _, key := range consensus.priKey { if key.Pub.Object.IsEqual(obj) { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 18feeae286..1fc26e7607 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -100,8 +100,8 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb canHandleViewChange := true intendedForValidator, intendedForLeader := - !consensus.IsLeader(), - consensus.IsLeader() + !consensus.isLeader(), + consensus.isLeader() // if in backup normal mode, force ignore view change event and leader event. if consensus.current.Mode() == NormalBackup { @@ -183,7 +183,7 @@ func (consensus *Consensus) finalCommit() { // Note: leader already sent 67% commit in preCommit. The 100% commit won't be sent immediately // to save network traffic. It will only be sent in retry if consensus doesn't move forward. // Or if the leader is changed for next block, the 100% committed sig will be sent to the next leader immediately. - if !consensus.IsLeader() || block.IsLastBlockInEpoch() { + if !consensus.isLeader() || block.IsLastBlockInEpoch() { // send immediately if err := consensus.msgSender.SendWithRetry( block.NumberU64(), @@ -248,7 +248,7 @@ func (consensus *Consensus) finalCommit() { // If still the leader, send commit sig/bitmap to finish the new block proposal, // else, the block proposal will timeout by itself. - if consensus.IsLeader() { + if consensus.isLeader() { if block.IsLastBlockInEpoch() { // No pipelining go func() { @@ -330,7 +330,7 @@ func (consensus *Consensus) Start( func (consensus *Consensus) StartChannel() { consensus.mutex.Lock() - consensus.isInitialLeader = consensus.IsLeader() + consensus.isInitialLeader = consensus.isLeader() if consensus.isInitialLeader { consensus.start = true consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Send ReadySignal") From 25a0655a4463ce6404da3a857e8695ae0176b5aa Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 00:14:19 +0700 Subject: [PATCH 231/420] Make ParseFBFTMessage private. --- consensus/consensus_v2.go | 2 +- consensus/construct.go | 2 +- consensus/fbft_log.go | 6 ++++++ consensus/validator.go | 6 +----- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 1fc26e7607..77beac988c 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -92,7 +92,7 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb members := consensus.Decider.Participants() fbftMsg, err = ParseNewViewMessage(msg, members) default: - fbftMsg, err = consensus.ParseFBFTMessage(msg) + fbftMsg, err = consensus.parseFBFTMessage(msg) } if err != nil || fbftMsg == nil { return errors.Wrapf(err, "unable to parse consensus msg with type: %s", msg.Type) diff --git a/consensus/construct.go b/consensus/construct.go index bbee71203a..8d9f3838aa 100644 --- a/consensus/construct.go +++ b/consensus/construct.go @@ -143,7 +143,7 @@ func (consensus *Consensus) construct( return nil, err } - FBFTMsg, err2 := consensus.ParseFBFTMessage(message) + FBFTMsg, err2 := consensus.parseFBFTMessage(message) if err2 != nil { utils.Logger().Error().Err(err). diff --git a/consensus/fbft_log.go b/consensus/fbft_log.go index 4743085ab6..ba37451038 100644 --- a/consensus/fbft_log.go +++ b/consensus/fbft_log.go @@ -287,6 +287,12 @@ func (log *FBFTLog) FindMessageByMaxViewID(msgs []*FBFTMessage) *FBFTMessage { // ParseFBFTMessage parses FBFT message into FBFTMessage structure func (consensus *Consensus) ParseFBFTMessage(msg *msg_pb.Message) (*FBFTMessage, error) { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + return consensus.parseFBFTMessage(msg) +} + +func (consensus *Consensus) parseFBFTMessage(msg *msg_pb.Message) (*FBFTMessage, error) { // TODO Have this do sanity checks on the message please pbftMsg := FBFTMessage{} pbftMsg.MessageType = msg.GetType() diff --git a/consensus/validator.go b/consensus/validator.go index b33ac6d688..cb2971e4cf 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -18,8 +18,7 @@ import ( ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { - - recvMsg, err := consensus.ParseFBFTMessage(msg) + recvMsg, err := consensus.parseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). Err(err). @@ -27,9 +26,6 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Msg("[OnAnnounce] Unparseable leader message") return } - if consensus.ShardID == 0 { - //fmt.Println("onAnnounce called ", recvMsg.BlockNum) - } // NOTE let it handle its own logs if !consensus.onAnnounceSanityChecks(recvMsg) { From fb3130eb1b1b67662cd31f9eba06aa9250ccf522 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 11:46:49 +0700 Subject: [PATCH 232/420] Fix remove locks. --- consensus/consensus_service.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index fdbcbc0457..4784b99fc0 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -429,8 +429,9 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { - consensus.mutex.RLock() - defer consensus.mutex.RUnlock() + // TODO: if remove locks blockchain stucks. + //consensus.mutex.RLock() + //defer consensus.mutex.RUnlock() return consensus.isLeader() } From c87e14be7864d8c29ed4eeafc4e9e65e66008288 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 11:58:01 +0700 Subject: [PATCH 233/420] Added additional locks. --- consensus/consensus_v2.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 77beac988c..a862d725ab 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -436,7 +436,8 @@ func (consensus *Consensus) BlockChannel(newBlock *types.Block) { consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") time.AfterFunc(time.Until(consensus.NextBlockDue), func() { consensus.StartFinalityCount() - + consensus.mutex.Lock() + defer consensus.mutex.Unlock() // Update time due for next block consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) From 70885aeb6a6e8f037e67409f9e9aab9631707c29 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 12:05:41 +0700 Subject: [PATCH 234/420] Added additional locks. --- consensus/validator.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/consensus/validator.go b/consensus/validator.go index cb2971e4cf..79e988fce4 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -271,6 +271,8 @@ func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { return } curBlockNum := consensus.BlockNum() + consensus.mutex.Lock() + defer consensus.mutex.Unlock() for _, committedMsg := range consensus.FBFTLog.GetNotVerifiedCommittedMessages(blockObj.NumberU64(), blockObj.Header().ViewID().Uint64(), blockObj.Hash()) { if committedMsg != nil { consensus.onCommitted(committedMsg) @@ -284,9 +286,6 @@ func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - consensus.getLogger().Info(). Uint64("MsgBlockNum", recvMsg.BlockNum). Uint64("MsgViewID", recvMsg.ViewID). From e581feb7f7e2d12d2896f260b7053e121b5f5207 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 12:22:26 +0700 Subject: [PATCH 235/420] Added readSignatureBitmapPayload locks. --- consensus/consensus_service.go | 15 ++++++++++----- consensus/validator.go | 4 ++-- consensus/view_change.go | 2 +- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 4784b99fc0..53a1a1a7c0 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -7,6 +7,7 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/crypto/bls" + "github.com/harmony-one/harmony/multibls" "github.com/ethereum/go-ethereum/common" protobuf "github.com/golang/protobuf/proto" @@ -229,16 +230,20 @@ func (consensus *Consensus) SetBlockNum(blockNum uint64) { } // ReadSignatureBitmapPayload read the payload for signature and bitmap; offset is the beginning position of reading -func (consensus *Consensus) ReadSignatureBitmapPayload( - recvPayload []byte, offset int, -) (*bls_core.Sign, *bls_cosi.Mask, error) { +func (consensus *Consensus) ReadSignatureBitmapPayload(recvPayload []byte, offset int) (*bls_core.Sign, *bls_cosi.Mask, error) { + consensus.mutex.RLock() + members := consensus.Decider.Participants() + consensus.mutex.RUnlock() + return consensus.readSignatureBitmapPayload(recvPayload, offset, members) +} + +func (consensus *Consensus) readSignatureBitmapPayload(recvPayload []byte, offset int, members multibls.PublicKeys) (*bls_core.Sign, *bls_cosi.Mask, error) { if offset+bls.BLSSignatureSizeInBytes > len(recvPayload) { return nil, nil, errors.New("payload not have enough length") } sigAndBitmapPayload := recvPayload[offset:] // TODO(audit): keep a Mask in the Decider so it won't be reconstructed on the fly. - members := consensus.Decider.Participants() return chain.ReadSignatureBitmapByPublicKeys( sigAndBitmapPayload, members, ) @@ -524,7 +529,7 @@ func (consensus *Consensus) selfCommit(payload []byte) error { return errGetPreparedBlock } - aggSig, mask, err := consensus.ReadSignatureBitmapPayload(payload, 32) + aggSig, mask, err := consensus.readSignatureBitmapPayload(payload, 32, consensus.Decider.Participants()) if err != nil { return errReadBitmapPayload } diff --git a/consensus/validator.go b/consensus/validator.go index 79e988fce4..7336c7b477 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -62,7 +62,7 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { if len(recvMsg.Block) > 0 { go func() { // Best effort check, no need to error out. - _, err := consensus.validateNewBlock(recvMsg) + _, err := consensus.ValidateNewBlock(recvMsg) if err == nil { consensus.getLogger().Info(). @@ -205,7 +205,7 @@ func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { // check validity of prepared signature blockHash := recvMsg.BlockHash - aggSig, mask, err := consensus.ReadSignatureBitmapPayload(recvMsg.Payload, 0) + aggSig, mask, err := consensus.readSignatureBitmapPayload(recvMsg.Payload, 0, consensus.Decider.Participants()) if err != nil { consensus.getLogger().Error().Err(err).Msg("ReadSignatureBitmapPayload failed!") return diff --git a/consensus/view_change.go b/consensus/view_change.go index 95ccbb97aa..9951cf06ba 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -501,7 +501,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { utils.CountOneBits(m3Mask.Bitmap) > utils.CountOneBits(m2Mask.Bitmap)) { // m1 is not empty, check it's valid blockHash := recvMsg.Payload[:32] - aggSig, mask, err := consensus.ReadSignatureBitmapPayload(recvMsg.Payload, 32) + aggSig, mask, err := consensus.readSignatureBitmapPayload(recvMsg.Payload, 32, consensus.Decider.Participants()) if err != nil { consensus.getLogger().Error().Err(err). Msg("[onNewView] ReadSignatureBitmapPayload Failed") From ba682fc90899834db31c622e7e4fc09a6373ceab Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 12:50:03 +0700 Subject: [PATCH 236/420] Added HandleMessageUpdate locks. --- consensus/consensus.go | 8 ++++---- consensus/consensus_service.go | 7 ++++++- consensus/consensus_v2.go | 8 +++++--- consensus/leader.go | 9 +-------- consensus/threshold.go | 2 +- consensus/view_change.go | 11 ++--------- 6 files changed, 19 insertions(+), 26 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 887b4744e9..11a0c29c89 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -200,7 +200,7 @@ func (consensus *Consensus) GetPrivateKeys() multibls.PrivateKeys { } // GetLeaderPrivateKey returns leader private key if node is the leader -func (consensus *Consensus) GetLeaderPrivateKey(leaderKey *bls_core.PublicKey) (*bls.PrivateKeyWrapper, error) { +func (consensus *Consensus) getLeaderPrivateKey(leaderKey *bls_core.PublicKey) (*bls.PrivateKeyWrapper, error) { for i, key := range consensus.priKey { if key.Pub.Object.IsEqual(leaderKey) { return &consensus.priKey[i], nil @@ -209,9 +209,9 @@ func (consensus *Consensus) GetLeaderPrivateKey(leaderKey *bls_core.PublicKey) ( return nil, errors.Wrapf(errLeaderPriKeyNotFound, leaderKey.SerializeToHexStr()) } -// GetConsensusLeaderPrivateKey returns consensus leader private key if node is the leader -func (consensus *Consensus) GetConsensusLeaderPrivateKey() (*bls.PrivateKeyWrapper, error) { - return consensus.GetLeaderPrivateKey(consensus.LeaderPubKey.Object) +// getConsensusLeaderPrivateKey returns consensus leader private key if node is the leader +func (consensus *Consensus) getConsensusLeaderPrivateKey() (*bls.PrivateKeyWrapper, error) { + return consensus.getLeaderPrivateKey(consensus.LeaderPubKey.Object) } // SetBlockVerifier sets the block verifier diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 53a1a1a7c0..d613f70004 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -468,7 +468,12 @@ func (consensus *Consensus) isLeader() bool { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { - fmt.Println("SetViewIDs", height) + consensus.setViewIDs(height) +} + +// SetViewIDs set both current view ID and view changing ID to the height +// of the blockchain. It is used during client startup to recover the state +func (consensus *Consensus) setViewIDs(height uint64) { consensus.SetCurBlockViewID(height) consensus.SetViewChangingID(height) } diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index a862d725ab..e866e1af71 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -56,6 +56,8 @@ func (consensus *Consensus) isViewChangingMode() bool { // HandleMessageUpdate will update the consensus state according to received message func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb.Message, senderKey *bls.SerializedPublicKey) error { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() // when node is in ViewChanging mode, it still accepts normal messages into FBFTLog // in order to avoid possible trap forever but drop PREPARE and COMMIT // which are message types specifically for a node acting as leader @@ -144,7 +146,7 @@ func (consensus *Consensus) finalCommit() { Msg("[finalCommit] Finalizing Consensus") beforeCatchupNum := consensus.BlockNum() - leaderPriKey, err := consensus.GetConsensusLeaderPrivateKey() + leaderPriKey, err := consensus.getConsensusLeaderPrivateKey() if err != nil { consensus.getLogger().Error().Err(err).Msg("[finalCommit] leader not found") return @@ -559,7 +561,7 @@ func (consensus *Consensus) preCommitAndPropose(blk *types.Block) error { return errors.New("block to pre-commit is nil") } - leaderPriKey, err := consensus.GetConsensusLeaderPrivateKey() + leaderPriKey, err := consensus.getConsensusLeaderPrivateKey() if err != nil { consensus.getLogger().Error().Err(err).Msg("[preCommitAndPropose] leader not found") return err @@ -804,7 +806,7 @@ func (consensus *Consensus) postCatchup(initBN uint64) { // GenerateVrfAndProof generates new VRF/Proof from hash of previous block func (consensus *Consensus) GenerateVrfAndProof(newHeader *block.Header) error { - key, err := consensus.GetConsensusLeaderPrivateKey() + key, err := consensus.getConsensusLeaderPrivateKey() if err != nil { return errors.New("[GenerateVrfAndProof] no leader private key provided") } diff --git a/consensus/leader.go b/consensus/leader.go index 6b176d27ae..548ec98c0a 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -29,7 +29,7 @@ func (consensus *Consensus) announce(block *types.Block) { copy(consensus.blockHash[:], blockHash[:]) consensus.block = encodedBlock // Must set block bytes before consensus.construct() - key, err := consensus.GetConsensusLeaderPrivateKey() + key, err := consensus.getConsensusLeaderPrivateKey() if err != nil { consensus.getLogger().Warn().Err(err).Msg("[Announce] Node not a leader") return @@ -102,11 +102,6 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { Uint64("MsgBlockNum", recvMsg.BlockNum). Msg("[OnPrepare] No Matching Announce message") } - - //// Read - Start - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - if !consensus.isRightBlockNumAndViewID(recvMsg) { return } @@ -195,8 +190,6 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { - consensus.mutex.Lock() - defer consensus.mutex.Unlock() //// Read - Start if !consensus.isRightBlockNumAndViewID(recvMsg) { return diff --git a/consensus/threshold.go b/consensus/threshold.go index e8bd875cba..ecb54980a0 100644 --- a/consensus/threshold.go +++ b/consensus/threshold.go @@ -15,7 +15,7 @@ import ( func (consensus *Consensus) didReachPrepareQuorum() error { logger := utils.Logger() logger.Info().Msg("[OnPrepare] Received Enough Prepare Signatures") - leaderPriKey, err := consensus.GetConsensusLeaderPrivateKey() + leaderPriKey, err := consensus.getConsensusLeaderPrivateKey() if err != nil { utils.Logger().Warn().Err(err).Msg("[OnPrepare] leader not found") return err diff --git a/consensus/view_change.go b/consensus/view_change.go index 9951cf06ba..720205ee83 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -347,10 +347,6 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri // onViewChange is called when the view change message is received. func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { - //fmt.Printf("[onViewChange] received view change message from %+v\n", recvMsg) - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - consensus.getLogger().Debug(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -359,7 +355,7 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { // if not leader, noop newLeaderKey := recvMsg.LeaderPubkey - newLeaderPriKey, err := consensus.GetLeaderPrivateKey(newLeaderKey.Object) + newLeaderPriKey, err := consensus.getLeaderPrivateKey(newLeaderKey.Object) if err != nil { consensus.getLogger().Debug(). Err(err). @@ -454,9 +450,6 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { // Or the validator will enter announce phase to wait for the new block proposed // from the new leader func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - consensus.getLogger().Info(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -543,7 +536,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.consensusTimeout[timeoutViewChange].Stop() // newView message verified success, override my state - consensus.SetViewIDs(recvMsg.ViewID) + consensus.setViewIDs(recvMsg.ViewID) consensus.LeaderPubKey = senderKey consensus.ResetViewChangeState() From 9cc517dac5b972bae36bbefebd333925f4b8a0c6 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 13:08:15 +0700 Subject: [PATCH 237/420] Added LastMile locks. --- api/service/legacysync/syncing.go | 26 +++++++++++++------------- consensus/consensus_v2.go | 10 +++++----- consensus/downloader.go | 29 ++++++++++++++--------------- 3 files changed, 32 insertions(+), 33 deletions(-) diff --git a/api/service/legacysync/syncing.go b/api/service/legacysync/syncing.go index 43f07c039b..3375ccdc50 100644 --- a/api/service/legacysync/syncing.go +++ b/api/service/legacysync/syncing.go @@ -18,6 +18,7 @@ import ( "github.com/harmony-one/harmony/api/service/legacysync/downloader" pb "github.com/harmony-one/harmony/api/service/legacysync/downloader/proto" "github.com/harmony-one/harmony/consensus" + consensus2 "github.com/harmony-one/harmony/consensus" "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" @@ -1142,20 +1143,19 @@ func (ss *StateSync) SyncLoop(bc core.BlockChain, worker *worker.Worker, isBeaco func (ss *StateSync) addConsensusLastMile(bc core.BlockChain, consensus *consensus.Consensus) error { curNumber := bc.CurrentBlock().NumberU64() - blockIter, err := consensus.GetLastMileBlockIter(curNumber + 1) - if err != nil { - return err - } - for { - block := blockIter.Next() - if block == nil { - break - } - if _, err := bc.InsertChain(types.Blocks{block}, true); err != nil { - return errors.Wrap(err, "failed to InsertChain") + err := consensus.GetLastMileBlockIter(curNumber+1, func(blockIter *consensus2.LastMileBlockIter) error { + for { + block := blockIter.Next() + if block == nil { + break + } + if _, err := bc.InsertChain(types.Blocks{block}, true); err != nil { + return errors.Wrap(err, "failed to InsertChain") + } } - } - return nil + return nil + }) + return err } // GetSyncingPort returns the syncing port. diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index e866e1af71..4253a46e95 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -496,24 +496,24 @@ type LastMileBlockIter struct { } // GetLastMileBlockIter get the iterator of the last mile blocks starting from number bnStart -func (consensus *Consensus) GetLastMileBlockIter(bnStart uint64) (*LastMileBlockIter, error) { +func (consensus *Consensus) GetLastMileBlockIter(bnStart uint64, cb func(iter *LastMileBlockIter) error) error { consensus.mutex.Lock() defer consensus.mutex.Unlock() if consensus.BlockVerifier == nil { - return nil, errors.New("consensus haven't initialized yet") + return errors.New("consensus haven't initialized yet") } blocks, _, err := consensus.getLastMileBlocksAndMsg(bnStart) if err != nil { - return nil, err + return err } - return &LastMileBlockIter{ + return cb(&LastMileBlockIter{ blockCandidates: blocks, fbftLog: consensus.FBFTLog, verify: consensus.BlockVerifier, curIndex: 0, logger: consensus.getLogger(), - }, nil + }) } // Next iterate to the next last mile block diff --git a/consensus/downloader.go b/consensus/downloader.go index 2734f8baed..8442ed5343 100644 --- a/consensus/downloader.go +++ b/consensus/downloader.go @@ -76,7 +76,7 @@ func (dh *downloadHelper) downloadFinishedLoop() { for { select { case <-dh.finishedCh: - err := dh.c.addConsensusLastMile() + err := dh.c.AddConsensusLastMile() if err != nil { dh.c.getLogger().Error().Err(err).Msg("add last mile failed") } @@ -89,22 +89,21 @@ func (dh *downloadHelper) downloadFinishedLoop() { } } -func (consensus *Consensus) addConsensusLastMile() error { +func (consensus *Consensus) AddConsensusLastMile() error { curBN := consensus.Blockchain().CurrentBlock().NumberU64() - blockIter, err := consensus.GetLastMileBlockIter(curBN + 1) - if err != nil { - return err - } - for { - block := blockIter.Next() - if block == nil { - break - } - if _, err := consensus.Blockchain().InsertChain(types.Blocks{block}, true); err != nil { - return errors.Wrap(err, "failed to InsertChain") + err := consensus.GetLastMileBlockIter(curBN+1, func(blockIter *LastMileBlockIter) error { + for { + block := blockIter.Next() + if block == nil { + break + } + if _, err := consensus.Blockchain().InsertChain(types.Blocks{block}, true); err != nil { + return errors.Wrap(err, "failed to InsertChain") + } } - } - return nil + return nil + }) + return err } func (consensus *Consensus) spinUpStateSync() { From 119c256e5f77d886bfb4a795db90b5b4bb6e933a Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 17:23:59 +0700 Subject: [PATCH 238/420] Locks for IsValidatorInCommittee. --- consensus/consensus_service.go | 6 ++++++ consensus/validator.go | 2 +- consensus/view_change.go | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index d613f70004..fdf0880625 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -159,6 +159,12 @@ func (consensus *Consensus) resetState() { // IsValidatorInCommittee returns whether the given validator BLS address is part of my committee func (consensus *Consensus) IsValidatorInCommittee(pubKey bls.SerializedPublicKey) bool { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() + return consensus.isValidatorInCommittee(pubKey) +} + +func (consensus *Consensus) isValidatorInCommittee(pubKey bls.SerializedPublicKey) bool { return consensus.Decider.IndexOf(pubKey) != -1 } diff --git a/consensus/validator.go b/consensus/validator.go index 7336c7b477..2ccc453645 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -408,7 +408,7 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { func (consensus *Consensus) getPriKeysInCommittee() []*bls.PrivateKeyWrapper { priKeys := []*bls.PrivateKeyWrapper{} for i, key := range consensus.priKey { - if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { + if !consensus.isValidatorInCommittee(key.Pub.Bytes) { continue } priKeys = append(priKeys, &consensus.priKey[i]) diff --git a/consensus/view_change.go b/consensus/view_change.go index 720205ee83..1a2accf727 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -279,7 +279,7 @@ func (consensus *Consensus) startViewChange() { // for view change, send separate view change per public key // do not do multi-sign of view change message for _, key := range consensus.priKey { - if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { + if !consensus.isValidatorInCommittee(key.Pub.Bytes) { continue } msgToSend := consensus.constructViewChangeMessage(&key) From 8f993111efa58e75927b1326886e6713e232c8e7 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 20:29:44 +0700 Subject: [PATCH 239/420] Fixed locks. --- consensus/consensus_msg_sender.go | 10 ++++++++-- consensus/consensus_service.go | 11 ++++++----- consensus/consensus_v2.go | 9 ++++++--- node/node_handler.go | 5 ++--- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/consensus/consensus_msg_sender.go b/consensus/consensus_msg_sender.go index 003d89cd96..ffa9d8b1c6 100644 --- a/consensus/consensus_msg_sender.go +++ b/consensus/consensus_msg_sender.go @@ -67,7 +67,10 @@ func (sender *MessageSender) SendWithRetry(blockNum uint64, msgType msg_pb.Messa sender.Retry(&msgRetry) }() } - return sender.host.SendMessageToGroups(groups, p2pMsg) + // MessageSender lays inside consensus, but internally calls consensus public api. + // Tt would be deadlock if run in current thread. + go sender.host.SendMessageToGroups(groups, p2pMsg) + return nil } // DelayedSendWithRetry is similar to SendWithRetry but without the initial message sending but only retries. @@ -86,7 +89,10 @@ func (sender *MessageSender) DelayedSendWithRetry(blockNum uint64, msgType msg_p // SendWithoutRetry sends message without retry logic. func (sender *MessageSender) SendWithoutRetry(groups []nodeconfig.GroupID, p2pMsg []byte) error { - return sender.host.SendMessageToGroups(groups, p2pMsg) + // MessageSender lays inside consensus, but internally calls consensus public api. + // It would be deadlock if run in current thread. + go sender.host.SendMessageToGroups(groups, p2pMsg) + return nil } // Retry will retry the consensus message for times. diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index fdf0880625..daaa219842 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -74,6 +74,8 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) consensus.getLogger().Info().Msg("My Committee updated") for i := range pubKeys { @@ -94,7 +96,7 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi } // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. - consensus.UpdateBitmaps() + consensus.updateBitmaps() consensus.resetState() // do not reset view change state if it is in view changing mode @@ -126,7 +128,7 @@ func (consensus *Consensus) signConsensusMessage(message *msg_pb.Message, } // UpdateBitmaps update the bitmaps for prepare and commit phase -func (consensus *Consensus) UpdateBitmaps() { +func (consensus *Consensus) updateBitmaps() { consensus.getLogger().Debug(). Str("MessageType", consensus.phase.String()). Msg("[UpdateBitmaps] Updating consensus bitmaps") @@ -440,9 +442,8 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { - // TODO: if remove locks blockchain stucks. - //consensus.mutex.RLock() - //defer consensus.mutex.RUnlock() + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() return consensus.isLeader() } diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 4253a46e95..37adeea430 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -703,11 +703,14 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess } consensus.FinishFinalityCount() - consensus.PostConsensusJob(blk) - consensus.SetupForNewConsensus(blk, committedMsg) + go func() { + consensus.PostConsensusJob(blk) + }() + consensus.setupForNewConsensus(blk, committedMsg) utils.Logger().Info().Uint64("blockNum", blk.NumberU64()). Str("hash", blk.Header().Hash().Hex()). Msg("Added New Block to Blockchain!!!") + return nil } @@ -756,7 +759,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { } // SetupForNewConsensus sets the state for new consensus -func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { +func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) consensus.SetCurBlockViewID(committedMsg.ViewID + 1) consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] diff --git a/node/node_handler.go b/node/node_handler.go index d15b41eb5a..3589d19660 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -361,9 +361,8 @@ func VerifyNewBlock(nodeConfig *nodeconfig.ConfigType, blockChain core.BlockChai } // PostConsensusProcessing is called by consensus participants, after consensus is done, to: -// 1. add the new block to blockchain -// 2. [leader] send new block to the client -// 3. [leader] send cross shard tx receipts to destination shard +// 1. [leader] send new block to the client +// 2. [leader] send cross shard tx receipts to destination shard func (node *Node) PostConsensusProcessing(newBlock *types.Block) error { if node.Consensus.IsLeader() { if node.IsRunningBeaconChain() { From 183ea26aacc56967e2eb2ae08a70f92df16b17e9 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 20:44:43 +0700 Subject: [PATCH 240/420] Fixed tests. --- consensus/construct_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/construct_test.go b/consensus/construct_test.go index 71da42e3f4..1add32219d 100644 --- a/consensus/construct_test.go +++ b/consensus/construct_test.go @@ -67,8 +67,8 @@ func TestConstructPreparedMessage(test *testing.T) { if err != nil { test.Fatalf("Cannot craeate consensus: %v", err) } - consensus.ResetState() - consensus.UpdateBitmaps() + consensus.resetState() + consensus.updateBitmaps() consensus.blockHash = [32]byte{} message := "test string" From b281f248e825231590c303ed6118d31e24340a8c Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 8 Nov 2022 01:05:23 +0700 Subject: [PATCH 241/420] Fixed tests. --- consensus/consensus_service.go | 6 +++++- internal/utils/singleton.go | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index daaa219842..e54f5edf21 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -442,7 +442,11 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { - consensus.mutex.RLock() + _ = utils.AssertNoLongerThan0(5*time.Second, func() error { + consensus.mutex.RLock() + return nil + }) + defer consensus.mutex.RUnlock() return consensus.isLeader() diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index 10101d7673..e58a0cc96d 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path" + "runtime/debug" "strconv" "sync" "time" @@ -212,3 +213,19 @@ func GetPort() int { } return 0 } + +func AssertNoLongerThan0[E any](t time.Duration, f func() E) E { + ch := make(chan E) + defer close(ch) + stack := debug.Stack() + go func() { + select { + case <-time.After(t): + panic("AssertNoLongerThan0: " + string(stack)) + case <-ch: + return + } + }() + + return f() +} From f8a64ed5ec5c825be54452c2ca2aca1c404c1e1e Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 8 Nov 2022 13:34:53 +0700 Subject: [PATCH 242/420] Fixed lock. --- consensus/consensus_service.go | 37 +++++++++++++++++++++++++++++----- consensus/consensus_v2.go | 10 ++++----- internal/utils/singleton.go | 16 +++++++++++++++ 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e54f5edf21..e539264dbe 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -76,6 +76,10 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { consensus.mutex.Lock() defer consensus.mutex.Unlock() + return consensus.updatePublicKeys(pubKeys, allowlist) +} + +func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { consensus.Decider.UpdateParticipants(pubKeys, allowlist) consensus.getLogger().Info().Msg("My Committee updated") for i := range pubKeys { @@ -172,6 +176,13 @@ func (consensus *Consensus) isValidatorInCommittee(pubKey bls.SerializedPublicKe // SetMode sets the mode of consensus func (consensus *Consensus) SetMode(m Mode) { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + consensus.setMode(m) +} + +// SetMode sets the mode of consensus +func (consensus *Consensus) setMode(m Mode) { if m == Normal && consensus.isBackup { m = NormalBackup } @@ -212,8 +223,8 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if consensus.IgnoreViewIDCheck.IsSet() { //in syncing mode, node accepts incoming messages without viewID/leaderKey checking //so only set mode to normal when new node enters consensus and need checking viewID - consensus.SetMode(Normal) - consensus.SetViewIDs(msg.ViewID) + consensus.setMode(Normal) + consensus.setViewIDs(msg.ViewID) if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } @@ -268,6 +279,12 @@ func (consensus *Consensus) readSignatureBitmapPayload(recvPayload []byte, offse // (b) node in committed but has any err during processing: Syncing mode // (c) node in committed and everything looks good: Normal mode func (consensus *Consensus) UpdateConsensusInformation() Mode { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + return consensus.updateConsensusInformation() +} + +func (consensus *Consensus) updateConsensusInformation() Mode { curHeader := consensus.Blockchain().CurrentHeader() curEpoch := curHeader.Epoch() nextEpoch := new(big.Int).Add(curHeader.Epoch(), common.Big1) @@ -372,7 +389,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { consensus.getLogger().Info(). Int("numPubKeys", len(pubKeys)). Msg("[UpdateConsensusInformation] Successfully updated public keys") - consensus.UpdatePublicKeys(pubKeys, shard.Schedule.InstanceForEpoch(nextEpoch).ExternalAllowlist()) + consensus.updatePublicKeys(pubKeys, shard.Schedule.InstanceForEpoch(nextEpoch).ExternalAllowlist()) // Update voters in the committee if _, err := consensus.Decider.SetVoters( @@ -485,8 +502,8 @@ func (consensus *Consensus) SetViewIDs(height uint64) { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) setViewIDs(height uint64) { - consensus.SetCurBlockViewID(height) - consensus.SetViewChangingID(height) + consensus.setCurBlockViewID(height) + consensus.setViewChangingID(height) } // SetCurBlockViewID set the current view ID @@ -494,11 +511,21 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } +// SetCurBlockViewID set the current view ID +func (consensus *Consensus) setCurBlockViewID(viewID uint64) { + consensus.current.SetCurBlockViewID(viewID) +} + // SetViewChangingID set the current view change ID func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) } +// SetViewChangingID set the current view change ID +func (consensus *Consensus) setViewChangingID(viewID uint64) { + consensus.current.SetViewChangingID(viewID) +} + // StartFinalityCount set the finality counter to current time func (consensus *Consensus) StartFinalityCount() { consensus.finalityCounter.Store(time.Now().UnixNano()) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 37adeea430..a30bc392a9 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -348,7 +348,7 @@ func (consensus *Consensus) syncReadyChan() { if consensus.BlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) consensus.SetViewIDs(consensus.Blockchain().CurrentHeader().ViewID().Uint64() + 1) - mode := consensus.UpdateConsensusInformation() + mode := consensus.updateConsensusInformation() consensus.current.SetMode(mode) consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") consensus.consensusTimeout[timeoutConsensus].Start() @@ -357,8 +357,8 @@ func (consensus *Consensus) syncReadyChan() { } else if consensus.Mode() == Syncing { // Corner case where sync is triggered before `onCommitted` and there is a race // for block insertion between consensus and downloader. - mode := consensus.UpdateConsensusInformation() - consensus.SetMode(mode) + mode := consensus.updateConsensusInformation() + consensus.setMode(mode) consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") consensus.consensusTimeout[timeoutConsensus].Start() consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() @@ -761,7 +761,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { // SetupForNewConsensus sets the state for new consensus func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) - consensus.SetCurBlockViewID(committedMsg.ViewID + 1) + consensus.setCurBlockViewID(committedMsg.ViewID + 1) consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] var epoch *big.Int if blk.IsLastBlockInEpoch() { @@ -775,7 +775,7 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg // Update consensus keys at last so the change of leader status doesn't mess up normal flow if blk.IsLastBlockInEpoch() { - consensus.SetMode(consensus.UpdateConsensusInformation()) + consensus.setMode(consensus.updateConsensusInformation()) } consensus.FBFTLog.PruneCacheBeforeBlock(blk.NumberU64()) consensus.resetState() diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index e58a0cc96d..b940fbfeb4 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -229,3 +229,19 @@ func AssertNoLongerThan0[E any](t time.Duration, f func() E) E { return f() } + +func AssertNoLongerThan(t time.Duration, f func()) { + ch := make(chan struct{}) + defer close(ch) + stack := debug.Stack() + go func() { + select { + case <-time.After(t): + panic("AssertNoLongerThan0: " + string(stack)) + case <-ch: + return + } + }() + + f() +} From 6883dd3e09aac012f4532997e2f755cab1855ddc Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 8 Nov 2022 19:30:57 +0700 Subject: [PATCH 243/420] Rebased over leader rotation. --- consensus/consensus.go | 8 +++++--- consensus/consensus_service.go | 18 +----------------- consensus/consensus_v2.go | 18 ++++-------------- consensus/view_change.go | 2 +- 4 files changed, 11 insertions(+), 35 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 11a0c29c89..10144c9108 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -178,6 +178,10 @@ func (consensus *Consensus) getPublicKeys() multibls.PublicKeys { } func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper { + return consensus.getLeaderPubKey() +} + +func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } @@ -185,10 +189,8 @@ func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } -func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { - consensus.pubKeyLock.Lock() +func (consensus *Consensus) setLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { consensus.LeaderPubKey = pub - consensus.pubKeyLock.Unlock() } func (consensus *Consensus) setLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e539264dbe..1062b4502a 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -459,11 +459,7 @@ func (consensus *Consensus) updateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { - _ = utils.AssertNoLongerThan0(5*time.Second, func() error { - consensus.mutex.RLock() - return nil - }) - + consensus.mutex.RLock() defer consensus.mutex.RUnlock() return consensus.isLeader() @@ -481,18 +477,6 @@ func (consensus *Consensus) isLeader() bool { return false } -// isLeader check if the node is a leader or not by comparing the public key of -// the node with the leader public key. This function assume it runs under lock. -func (consensus *Consensus) isLeader() bool { - obj := consensus.LeaderPubKey.Object - for _, key := range consensus.priKey { - if key.Pub.Object.IsEqual(obj) { - return true - } - } - return false -} - // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index a30bc392a9..b8696200c3 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -460,16 +460,6 @@ func (consensus *Consensus) BlockChannel(newBlock *types.Block) { } } -// Close closes the consensus. If current is in normal commit phase, wait until the commit -// phase end. -func (consensus *Consensus) Close() error { - if consensus.dHelper != nil { - consensus.dHelper.close() - } - consensus.waitForCommit() - return nil -} - // waitForCommit wait extra 2 seconds for commit phase to finish func (consensus *Consensus) waitForCommit() { if consensus.Mode() != Normal || consensus.phase.Get() != FBFTCommit { @@ -717,11 +707,11 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess // rotateLeader rotates the leader to the next leader in the committee. // This function must be called with enabled leader rotation. func (consensus *Consensus) rotateLeader(epoch *big.Int) { - prev := consensus.GetLeaderPubKey() + prev := consensus.getLeaderPubKey() bc := consensus.Blockchain() curNumber := bc.CurrentHeader().Number().Uint64() utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), bc.Config().IsLeaderRotation(epoch), bc.Config().LeaderRotationBlocksCount) - leader := consensus.GetLeaderPubKey() + leader := consensus.getLeaderPubKey() for i := 0; i < bc.Config().LeaderRotationBlocksCount; i++ { header := bc.GetHeaderByNumber(curNumber - uint64(i)) if header == nil { @@ -748,9 +738,9 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { utils.Logger().Error().Msg("Failed to get next leader") return } else { - consensus.SetLeaderPubKey(next) + consensus.setLeaderPubKey(next) } - if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { + if consensus.isLeader() && !consensus.getLeaderPubKey().Object.IsEqual(prev.Object) { // leader changed go func() { consensus.ReadySignal <- SyncProposal diff --git a/consensus/view_change.go b/consensus/view_change.go index 1a2accf727..d7080c5a79 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -340,7 +340,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.resetState() } - consensus.SetLeaderPubKey(newLeaderPriKey.Pub) + consensus.setLeaderPubKey(newLeaderPriKey.Pub) return nil } From b9f71e31c1b37aed8f04903c77196171592d0282 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Fri, 18 Nov 2022 18:24:48 +0700 Subject: [PATCH 244/420] Fix formatting. --- internal/params/config.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/internal/params/config.go b/internal/params/config.go index f1000098a3..c86d5ce88c 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -319,8 +319,6 @@ var ( big.NewInt(0), // AllowlistEpoch big.NewInt(1), // LeaderRotationEpoch 64, // LeaderRotationBlocksCount - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } @@ -361,9 +359,8 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } From 192a701a40beb53f45c5208e82d6d107376d508f Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 17 Jan 2023 12:55:50 -0300 Subject: [PATCH 245/420] Rebased onto dev. --- internal/params/config.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/internal/params/config.go b/internal/params/config.go index c86d5ce88c..725704bd06 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -111,7 +111,6 @@ var ( AllowlistEpoch: big.NewInt(2), LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 64, - TesnetNinetyPercentEpoch: big.NewInt(399), FeeCollectEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. @@ -274,7 +273,6 @@ var ( AllowlistEpoch: EpochTBD, LeaderRotationEpoch: big.NewInt(4), LeaderRotationBlocksCount: 5, - TesnetNinetyPercentEpoch: EpochTBD, FeeCollectEpoch: big.NewInt(5), LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 5, @@ -719,10 +717,6 @@ func (c *ChainConfig) IsLeaderRotation(epoch *big.Int) bool { return isForked(c.LeaderRotationEpoch, epoch) } -func (c *ChainConfig) IsTestnetNinetyPercent(epoch *big.Int) bool { - return isForked(c.TesnetNinetyPercentEpoch, epoch) && c == TestnetChainConfig -} - // IsFeeCollectEpoch determines whether Txn Fees will be collected into the community-managed account. func (c *ChainConfig) IsFeeCollectEpoch(epoch *big.Int) bool { return isForked(c.FeeCollectEpoch, epoch) From 07a38ac3b7f5bc8d70e18ec8b62bad1cf745fddf Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 1 Sep 2022 00:53:16 +0700 Subject: [PATCH 246/420] in progress. --- api/service/explorer/service.go | 8 +++++--- cmd/harmony/main.go | 3 +++ consensus/consensus.go | 2 ++ consensus/consensus_service.go | 29 +++++++++++++++++++++++++++++ consensus/consensus_v2.go | 17 ++++++++++------- consensus/leader.go | 1 + consensus/quorum/quorum.go | 7 ++++++- consensus/validator.go | 4 ++++ consensus/view_change.go | 8 ++++++++ internal/utils/singleton.go | 33 +-------------------------------- node/node_newblock.go | 2 -- 11 files changed, 69 insertions(+), 45 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 6aa275332b..d96c1bc56c 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -115,6 +115,8 @@ func (s *Service) Run() *http.Server { s.router = mux.NewRouter() + fmt.Println("++", addr) + // Set up router for addresses. // Fetch addresses request, accepts parameter size: how much addresses to read, // parameter prefix: from which address prefix start @@ -221,9 +223,9 @@ func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { for i := cur; i > 0; i-- { block := s.blockchain.GetBlockByNumber(i) - w.Write([]byte(fmt.Sprintf("#%d ", i))) - w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) - w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) + w.Write([]byte(fmt.Sprintf("%d ", i))) + w.Write([]byte(fmt.Sprintf("%s ", block.Header().ViewID().String()))) + w.Write([]byte(fmt.Sprintf("%s ", block.Header().Coinbase().Hash().Hex()))) w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) } } diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 4bb70a81b4..2fcfe5f037 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -266,6 +266,8 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { var err error + fmt.Println("OS: ", os.Args) + nodeconfigSetShardSchedule(hc) nodeconfig.SetShardingSchedule(shard.Schedule) nodeconfig.SetVersion(getHarmonyVersion()) @@ -801,6 +803,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // Set the consensus ID to be the current block number viewID := currentNode.Blockchain().CurrentBlock().Header().ViewID().Uint64() + fmt.Println("viewID:", viewID) currentConsensus.SetViewIDs(viewID + 1) utils.Logger().Info(). Uint64("viewID", viewID). diff --git a/consensus/consensus.go b/consensus/consensus.go index 10144c9108..b73ecfb8e9 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -71,6 +71,8 @@ type Consensus struct { priKey multibls.PrivateKeys // the publickey of leader LeaderPubKey *bls.PublicKeyWrapper + // index of leader in the list of validators. + LeaderIndex int // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 1062b4502a..2fabb42b3c 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync/atomic" "time" @@ -228,6 +229,7 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } + fmt.Println("[checkViewID] Set LEADEER PUB KEY ", msg.SenderPubkeys[0].Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() @@ -477,9 +479,22 @@ func (consensus *Consensus) isLeader() bool { return false } +// isLeader check if the node is a leader or not by comparing the public key of +// the node with the leader public key. This function assume it runs under lock. +func (consensus *Consensus) isLeader() bool { + obj := consensus.LeaderPubKey.Object + for _, key := range consensus.priKey { + if key.Pub.Object.IsEqual(obj) { + return true + } + } + return false +} + // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { + fmt.Println("SetViewIDs", height) consensus.setViewIDs(height) } @@ -495,6 +510,19 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } +// SetLeaderIndex set the leader index. +func (consensus *Consensus) SetLeaderIndex(f func(int) int) (current int) { + consensus.pubKeyLock.Lock() + defer consensus.pubKeyLock.Unlock() + consensus.LeaderIndex = f(consensus.LeaderIndex) + return consensus.LeaderIndex +} + +func (consensus *Consensus) GetLeaderIndex() int { + consensus.pubKeyLock.Lock() + defer consensus.pubKeyLock.Unlock() + return consensus.LeaderIndex +} // SetCurBlockViewID set the current view ID func (consensus *Consensus) setCurBlockViewID(viewID uint64) { consensus.current.SetCurBlockViewID(viewID) @@ -505,6 +533,7 @@ func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) } + // SetViewChangingID set the current view change ID func (consensus *Consensus) setViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index b8696200c3..fea7b7dfe6 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -425,18 +425,20 @@ func (consensus *Consensus) Close() error { } func (consensus *Consensus) BlockChannel(newBlock *types.Block) { - //consensus.ReshardingNextLeader(newBlock)consensus.getLogger().Info(). - Uint64("MsgBlockNum", newBlock.NumberU64()). - Msg("[ConsensusMainLoop] Received Proposed New Block!") + //consensus.ReshardingNextLeader(newBlock) + consensus.getLogger().Info(). + Uint64("MsgBlockNum", newBlock.NumberU64()). + Msg("[ConsensusMainLoop] Received Proposed New Block!") if newBlock.NumberU64() < consensus.BlockNum() { consensus.getLogger().Warn().Uint64("newBlockNum", newBlock.NumberU64()). Msg("[ConsensusMainLoop] received old block, abort") return - } - // Sleep to wait for the full block time - consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") - time.AfterFunc(time.Until(consensus.NextBlockDue), func() { + } + // Sleep to wait for the full block time + consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") + + time.AfterFunc(time.Until(consensus.NextBlockDue), func() { consensus.StartFinalityCount() consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -664,6 +666,7 @@ func (consensus *Consensus) tryCatchup() error { consensus.getLogger().Error().Err(err).Msg("[TryCatchup] Failed to add block to chain") return err } + //fmt.Println("tryCatchup ", utils.GetPort(), blk.NumberU64()) select { // TODO: Remove this when removing dns sync and stream sync is fully up case consensus.VerifiedNewBlock <- blk: diff --git a/consensus/leader.go b/consensus/leader.go index 548ec98c0a..b34757ae64 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -322,4 +322,5 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED) } + //fmt.Println("onCommit99: ", utils.GetPort(), recvMsg.BlockNum) } diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 6cf79e549a..0586ace4fe 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -230,12 +230,17 @@ func (s *cIdentities) NthNextHmy(instance shardingconfig.Instance, pubKey *bls.P Msg("[NthNextHmy] pubKey not found") } numNodes := instance.NumHarmonyOperatedNodesPerShard() + //fmt.Println("??idx:", idx, numNodes) // sanity check to avoid out of bound access if numNodes <= 0 || numNodes > len(s.publicKeys) { numNodes = len(s.publicKeys) } idx = (idx + next) % numNodes - return found, &s.publicKeys[idx] + //fmt.Println("-------idx:", idx) + + new := &s.publicKeys[idx] + fmt.Println("NthNextHmy: ", pubKey.Bytes.Hex(), new.Bytes.Hex()) + return found, new } // NthNextHmyExt return the Nth next pubkey of Harmony + allowlist nodes, next can be negative number diff --git a/consensus/validator.go b/consensus/validator.go index 2ccc453645..2b18003212 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -18,6 +18,7 @@ import ( ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { + recvMsg, err := consensus.parseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). @@ -26,6 +27,9 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Msg("[OnAnnounce] Unparseable leader message") return } + if consensus.ShardID == 0 { + //fmt.Println("onAnnounce called ", recvMsg.BlockNum) + } // NOTE let it handle its own logs if !consensus.onAnnounceSanityChecks(recvMsg) { diff --git a/consensus/view_change.go b/consensus/view_change.go index d7080c5a79..fb1b995f48 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "time" @@ -141,6 +142,7 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { Uint64("stuckBlockViewID", stuckBlockViewID). Msg("[getNextViewID]") + fmt.Println("end getNextViewID: ", nextViewID, viewChangeDuration) // duration is always the fixed view change duration for synchronous view change return nextViewID, viewChangeDuration } @@ -213,6 +215,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe lastLeaderPubKey, gap) } + fmt.Println("wasfoundNext", consensus.Blockchain.Config().IsAllowlistEpoch(epoch), wasFound, next.Bytes.Hex(), lastLeaderPubKey.Bytes.Hex()) if !wasFound { consensus.getLogger().Warn(). Str("key", consensus.LeaderPubKey.Bytes.Hex()). @@ -234,6 +237,7 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { + fmt.Printf("Message to send leader111: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } @@ -242,6 +246,7 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() + //fmt.Println("startViewChange", nextViewID) consensus.SetViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet @@ -282,7 +287,9 @@ func (consensus *Consensus) startViewChange() { if !consensus.isValidatorInCommittee(key.Pub.Bytes) { continue } + // Тут уже другой leader msgToSend := consensus.constructViewChangeMessage(&key) + fmt.Println("Message to send leader222: ", consensus.LeaderPubKey.Bytes.Hex()) if err := consensus.msgSender.SendWithRetry( consensus.BlockNum(), msg_pb.MessageType_VIEWCHANGE, @@ -340,6 +347,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.resetState() } + fmt.Println("[startNewView]", newLeaderPriKey.Pub.Bytes.Hex()) consensus.setLeaderPubKey(newLeaderPriKey.Pub) return nil diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index b940fbfeb4..592e8dc6b3 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -8,6 +8,7 @@ import ( "path" "runtime/debug" "strconv" + "strconv" "sync" "time" @@ -213,35 +214,3 @@ func GetPort() int { } return 0 } - -func AssertNoLongerThan0[E any](t time.Duration, f func() E) E { - ch := make(chan E) - defer close(ch) - stack := debug.Stack() - go func() { - select { - case <-time.After(t): - panic("AssertNoLongerThan0: " + string(stack)) - case <-ch: - return - } - }() - - return f() -} - -func AssertNoLongerThan(t time.Duration, f func()) { - ch := make(chan struct{}) - defer close(ch) - stack := debug.Stack() - go func() { - select { - case <-time.After(t): - panic("AssertNoLongerThan0: " + string(stack)) - case <-ch: - return - } - }() - - f() -} diff --git a/node/node_newblock.go b/node/node_newblock.go index c29fb1b96a..69523861f2 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -89,8 +89,6 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp newBlock, err := node.ProposeNewBlock(newCommitSigsChan) if err == nil { - fmt.Printf("ProposeNewBlock: #%d :%d e:%d, sh:%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), node.Blockchain().CurrentHeader().Epoch().Uint64(), node.Blockchain().ShardID(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) - if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). Msg("Block with the same number was already proposed, abort.") From 6703feb5d8a2331cb9151c97247be17e7edbd280 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 5 Sep 2022 19:39:49 +0700 Subject: [PATCH 247/420] consensus check is forked --- api/service/explorer/service.go | 6 +++--- consensus/consensus.go | 2 -- internal/params/config.go | 2 -- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index d96c1bc56c..5887bb3a02 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -223,9 +223,9 @@ func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { for i := cur; i > 0; i-- { block := s.blockchain.GetBlockByNumber(i) - w.Write([]byte(fmt.Sprintf("%d ", i))) - w.Write([]byte(fmt.Sprintf("%s ", block.Header().ViewID().String()))) - w.Write([]byte(fmt.Sprintf("%s ", block.Header().Coinbase().Hash().Hex()))) + w.Write([]byte(fmt.Sprintf("#%d ", i))) + w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) + w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) } } diff --git a/consensus/consensus.go b/consensus/consensus.go index b73ecfb8e9..10144c9108 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -71,8 +71,6 @@ type Consensus struct { priKey multibls.PrivateKeys // the publickey of leader LeaderPubKey *bls.PublicKeyWrapper - // index of leader in the list of validators. - LeaderIndex int // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 diff --git a/internal/params/config.go b/internal/params/config.go index 725704bd06..f5ce665b1f 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -274,8 +274,6 @@ var ( LeaderRotationEpoch: big.NewInt(4), LeaderRotationBlocksCount: 5, FeeCollectEpoch: big.NewInt(5), - LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 5, } // AllProtocolChanges ... From 2381d0adda6e1bb9500ae9306732d6c4d94016a3 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 5 Sep 2022 22:41:06 +0700 Subject: [PATCH 248/420] update master --- api/service/explorer/service.go | 2 ++ consensus/view_change.go | 2 +- node/node_newblock.go | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 5887bb3a02..1ebfa94ba1 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -223,6 +223,8 @@ func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { for i := cur; i > 0; i-- { block := s.blockchain.GetBlockByNumber(i) + leaderPubKey, _ := chain.GetLeaderPubKeyFromCoinbase(s.backend.Blockchain(), block.Header()) + w.Write([]byte(fmt.Sprintf("%s ", leaderPubKey.Bytes.Hex()))) w.Write([]byte(fmt.Sprintf("#%d ", i))) w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) diff --git a/consensus/view_change.go b/consensus/view_change.go index fb1b995f48..226db683c4 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -237,7 +237,7 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - fmt.Printf("Message to send leader111: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) + fmt.Printf("[startViewChange]: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } diff --git a/node/node_newblock.go b/node/node_newblock.go index 69523861f2..f96477413f 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -146,7 +146,6 @@ func (node *Node) ProposeNewBlock(commitSigs chan []byte) (*types.Block, error) if node.Blockchain().Config().IsStaking(header.Epoch()) { blsPubKeyBytes := leaderKey.Object.GetAddress() coinbase.SetBytes(blsPubKeyBytes[:]) - fmt.Println("coinbase.SetBytes leader: ", leaderKey.Bytes.Hex(), coinbase.Hex()) } emptyAddr := common.Address{} From a34e4c84ccc520d41ce5f7df7a22b0607804b4f5 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 6 Sep 2022 02:58:48 +0700 Subject: [PATCH 249/420] fix leader --- consensus/consensus.go | 1 + consensus/consensus_service.go | 34 ++++++++++++++++++++++++++-------- consensus/consensus_v2.go | 2 +- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 10144c9108..33f6ee5a14 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -74,6 +74,7 @@ type Consensus struct { // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 + epoch uint64 // Blockhash - 32 byte blockHash [32]byte // Block to run consensus on diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 2fabb42b3c..76e5d9c44b 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -82,6 +82,7 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { consensus.Decider.UpdateParticipants(pubKeys, allowlist) + consensus.pubKeyLock.Unlock() consensus.getLogger().Info().Msg("My Committee updated") for i := range pubKeys { consensus.getLogger().Info(). @@ -89,15 +90,22 @@ func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.Publi Str("BLSPubKey", pubKeys[i].Bytes.Hex()). Msg("Member") } - - allKeys := consensus.Decider.Participants() - if len(allKeys) != 0 { - consensus.LeaderPubKey = &allKeys[0] - consensus.getLogger().Info(). - Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") + if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { + consensus.updateLeader() } else { - consensus.getLogger().Error(). - Msg("[UpdatePublicKeys] Participants is empty") + consensus.pubKeyLock.Lock() + allKeys := consensus.Decider.Participants() + consensus.pubKeyLock.Unlock() + if len(allKeys) != 0 { + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = &allKeys[0] + consensus.pubKeyLock.Unlock() + consensus.getLogger().Info(). + Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") + } else { + consensus.getLogger().Error(). + Msg("[UpdatePublicKeys] Participants is empty") + } } // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. @@ -658,3 +666,13 @@ func (consensus *Consensus) getLogger() *zerolog.Logger { Logger() return &logger } + +func UpdatePublicKeyDefault(consensus *Consensus) { + if allKeys := consensus.Decider.Participants(); len(allKeys) > 0 { + consensus.LeaderPubKey = &allKeys[0] + } +} + +func UpdatePublicKeyRotate(consensus *Consensus) { + //consensus +} diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index fea7b7dfe6..653ae72d66 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -778,7 +778,7 @@ func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, er if epoch.Uint64() == 0 { return 0, nil } - epochBlock := consensus.Blockchain.GetBlockByNumber(epoch.Uint64() - 1) + epochBlock := consensus.Blockchain.GetBlockByNumber(shard.Schedule.EpochLastBlock(epoch.Uint64() - 1)) if epochBlock == nil { return 0, errors.Errorf("block not found for number %d", epoch.Uint64()-1) } From e396b4fb93d4fed5dbc8992bfbc8f269cde41212 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 7 Sep 2022 22:20:05 +0700 Subject: [PATCH 250/420] check leader for N blocks --- consensus/consensus.go | 5 +++++ consensus/consensus_service.go | 31 +++++++++++++------------------ consensus/consensus_v2.go | 4 +--- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 33f6ee5a14..d7b86f1421 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -185,6 +185,11 @@ func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper { func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } +func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = pub + consensus.pubKeyLock.Unlock() +} func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 76e5d9c44b..5e553f2628 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -82,31 +82,26 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { consensus.Decider.UpdateParticipants(pubKeys, allowlist) + allKeys := consensus.Decider.Participants() consensus.pubKeyLock.Unlock() - consensus.getLogger().Info().Msg("My Committee updated") + if len(allKeys) != 0 { + first := consensus.Decider.FirstParticipant( + shard.Schedule.InstanceForEpoch(consensus.Blockchain.CurrentHeader().Epoch())) + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = first + consensus.pubKeyLock.Unlock() + consensus.getLogger().Info(). + Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") + } else { + consensus.getLogger().Error(). + Msg("[UpdatePublicKeys] Participants is empty") + } for i := range pubKeys { consensus.getLogger().Info(). Int("index", i). Str("BLSPubKey", pubKeys[i].Bytes.Hex()). Msg("Member") } - if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { - consensus.updateLeader() - } else { - consensus.pubKeyLock.Lock() - allKeys := consensus.Decider.Participants() - consensus.pubKeyLock.Unlock() - if len(allKeys) != 0 { - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = &allKeys[0] - consensus.pubKeyLock.Unlock() - consensus.getLogger().Info(). - Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") - } else { - consensus.getLogger().Error(). - Msg("[UpdatePublicKeys] Participants is empty") - } - } // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. consensus.updateBitmaps() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 653ae72d66..8a3d4110b2 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/hex" - "fmt" "math/big" "sync/atomic" "time" @@ -12,6 +11,7 @@ import ( "github.com/ethereum/go-ethereum/common" bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" + "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" @@ -437,7 +437,6 @@ func (consensus *Consensus) BlockChannel(newBlock *types.Block) { } // Sleep to wait for the full block time consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") - time.AfterFunc(time.Until(consensus.NextBlockDue), func() { consensus.StartFinalityCount() consensus.mutex.Lock() @@ -548,7 +547,6 @@ func (consensus *Consensus) getLastMileBlocksAndMsg(bnStart uint64) ([]*types.Bl // preCommitAndPropose commit the current block with 67% commit signatures and start // proposing new block which will wait on the full commit signatures to finish func (consensus *Consensus) preCommitAndPropose(blk *types.Block) error { - //fmt.Println("preCommitAndPropose", utils.GetPort(), blk.NumberU64()) if blk == nil { return errors.New("block to pre-commit is nil") } From 45e9bce15e7f3273a8867c43a3ce0b6420c270e8 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sat, 10 Sep 2022 18:02:45 +0700 Subject: [PATCH 251/420] fix --- api/service/explorer/service.go | 2 -- cmd/harmony/main.go | 3 --- consensus/consensus_service.go | 11 ----------- consensus/quorum/quorum.go | 7 +------ 4 files changed, 1 insertion(+), 22 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 1ebfa94ba1..b13f73db61 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -115,8 +115,6 @@ func (s *Service) Run() *http.Server { s.router = mux.NewRouter() - fmt.Println("++", addr) - // Set up router for addresses. // Fetch addresses request, accepts parameter size: how much addresses to read, // parameter prefix: from which address prefix start diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 2fcfe5f037..4bb70a81b4 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -266,8 +266,6 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { var err error - fmt.Println("OS: ", os.Args) - nodeconfigSetShardSchedule(hc) nodeconfig.SetShardingSchedule(shard.Schedule) nodeconfig.SetVersion(getHarmonyVersion()) @@ -803,7 +801,6 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // Set the consensus ID to be the current block number viewID := currentNode.Blockchain().CurrentBlock().Header().ViewID().Uint64() - fmt.Println("viewID:", viewID) currentConsensus.SetViewIDs(viewID + 1) utils.Logger().Info(). Uint64("viewID", viewID). diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 5e553f2628..3f75c0bb00 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -232,7 +232,6 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } - fmt.Println("[checkViewID] Set LEADEER PUB KEY ", msg.SenderPubkeys[0].Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() @@ -661,13 +660,3 @@ func (consensus *Consensus) getLogger() *zerolog.Logger { Logger() return &logger } - -func UpdatePublicKeyDefault(consensus *Consensus) { - if allKeys := consensus.Decider.Participants(); len(allKeys) > 0 { - consensus.LeaderPubKey = &allKeys[0] - } -} - -func UpdatePublicKeyRotate(consensus *Consensus) { - //consensus -} diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 0586ace4fe..6cf79e549a 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -230,17 +230,12 @@ func (s *cIdentities) NthNextHmy(instance shardingconfig.Instance, pubKey *bls.P Msg("[NthNextHmy] pubKey not found") } numNodes := instance.NumHarmonyOperatedNodesPerShard() - //fmt.Println("??idx:", idx, numNodes) // sanity check to avoid out of bound access if numNodes <= 0 || numNodes > len(s.publicKeys) { numNodes = len(s.publicKeys) } idx = (idx + next) % numNodes - //fmt.Println("-------idx:", idx) - - new := &s.publicKeys[idx] - fmt.Println("NthNextHmy: ", pubKey.Bytes.Hex(), new.Bytes.Hex()) - return found, new + return found, &s.publicKeys[idx] } // NthNextHmyExt return the Nth next pubkey of Harmony + allowlist nodes, next can be negative number From 7494703cd075b947ac63c61e7bb4667dc313dd92 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 14:04:09 +0700 Subject: [PATCH 252/420] fix --- consensus/consensus_service.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 3f75c0bb00..d436ef5d56 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync/atomic" "time" @@ -496,7 +495,6 @@ func (consensus *Consensus) isLeader() bool { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { - fmt.Println("SetViewIDs", height) consensus.setViewIDs(height) } From d13ef3a62e367239a2ad07bdd1bbb04abab2dccb Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 15:27:56 +0700 Subject: [PATCH 253/420] Cleanup and fix update pub keys. --- consensus/consensus_service.go | 16 ++++++++++------ consensus/consensus_v2.go | 1 - consensus/leader.go | 1 - consensus/view_change.go | 7 ------- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index d436ef5d56..1913460eb9 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync/atomic" "time" @@ -81,14 +82,17 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { consensus.Decider.UpdateParticipants(pubKeys, allowlist) + consensus.getLogger().Info().Msg("My Committee updated") + for i := range pubKeys { + consensus.getLogger().Info(). + Int("index", i). + Str("BLSPubKey", pubKeys[i].Bytes.Hex()). + Msg("Member") + } + allKeys := consensus.Decider.Participants() - consensus.pubKeyLock.Unlock() if len(allKeys) != 0 { - first := consensus.Decider.FirstParticipant( - shard.Schedule.InstanceForEpoch(consensus.Blockchain.CurrentHeader().Epoch())) - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = first - consensus.pubKeyLock.Unlock() + consensus.LeaderPubKey = &allKeys[0] consensus.getLogger().Info(). Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") } else { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 8a3d4110b2..3ec460ef52 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -664,7 +664,6 @@ func (consensus *Consensus) tryCatchup() error { consensus.getLogger().Error().Err(err).Msg("[TryCatchup] Failed to add block to chain") return err } - //fmt.Println("tryCatchup ", utils.GetPort(), blk.NumberU64()) select { // TODO: Remove this when removing dns sync and stream sync is fully up case consensus.VerifiedNewBlock <- blk: diff --git a/consensus/leader.go b/consensus/leader.go index b34757ae64..548ec98c0a 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -322,5 +322,4 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED) } - //fmt.Println("onCommit99: ", utils.GetPort(), recvMsg.BlockNum) } diff --git a/consensus/view_change.go b/consensus/view_change.go index 226db683c4..377a7098f7 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "time" @@ -142,7 +141,6 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { Uint64("stuckBlockViewID", stuckBlockViewID). Msg("[getNextViewID]") - fmt.Println("end getNextViewID: ", nextViewID, viewChangeDuration) // duration is always the fixed view change duration for synchronous view change return nextViewID, viewChangeDuration } @@ -215,7 +213,6 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe lastLeaderPubKey, gap) } - fmt.Println("wasfoundNext", consensus.Blockchain.Config().IsAllowlistEpoch(epoch), wasFound, next.Bytes.Hex(), lastLeaderPubKey.Bytes.Hex()) if !wasFound { consensus.getLogger().Warn(). Str("key", consensus.LeaderPubKey.Bytes.Hex()). @@ -237,7 +234,6 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - fmt.Printf("[startViewChange]: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } @@ -246,7 +242,6 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() - //fmt.Println("startViewChange", nextViewID) consensus.SetViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet @@ -287,9 +282,7 @@ func (consensus *Consensus) startViewChange() { if !consensus.isValidatorInCommittee(key.Pub.Bytes) { continue } - // Тут уже другой leader msgToSend := consensus.constructViewChangeMessage(&key) - fmt.Println("Message to send leader222: ", consensus.LeaderPubKey.Bytes.Hex()) if err := consensus.msgSender.SendWithRetry( consensus.BlockNum(), msg_pb.MessageType_VIEWCHANGE, From 1c190b8efc4f19f027c43926a6c041d0ab331c4b Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 16:32:36 +0700 Subject: [PATCH 254/420] Rotate leader. --- consensus/consensus.go | 1 - 1 file changed, 1 deletion(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index d7b86f1421..4cf4fdf6f8 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -74,7 +74,6 @@ type Consensus struct { // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 - epoch uint64 // Blockhash - 32 byte blockHash [32]byte // Block to run consensus on From 5874c3082c676469b7d2e310ea0540f35cea84a1 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 13 Sep 2022 01:10:13 +0700 Subject: [PATCH 255/420] fix fix fix fix fix --- consensus/consensus_service.go | 1 - consensus/consensus_v2.go | 6 ++++++ internal/params/config.go | 11 +++++++---- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 1913460eb9..411b9f6552 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync/atomic" "time" diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 3ec460ef52..e8fa0fb17c 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -759,6 +759,12 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg } else { epoch = blk.Epoch() } + var epoch *big.Int + if blk.IsLastBlockInEpoch() { + epoch = new(big.Int).Add(blk.Epoch(), common.Big1) + } else { + epoch = blk.Epoch() + } if consensus.Blockchain.Config().IsLeaderRotation(epoch) { consensus.rotateLeader(epoch) } diff --git a/internal/params/config.go b/internal/params/config.go index f5ce665b1f..310172dc71 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -274,6 +274,8 @@ var ( LeaderRotationEpoch: big.NewInt(4), LeaderRotationBlocksCount: 5, FeeCollectEpoch: big.NewInt(5), + LeaderRotationEpoch: big.NewInt(4), + LeaderRotationBlocksCount: 5, } // AllProtocolChanges ... @@ -313,8 +315,8 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } @@ -355,8 +357,9 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount + + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } From 37d82f99ade968ed6f5018e9eaf4adf1baeeec98 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 13 Sep 2022 02:15:25 +0700 Subject: [PATCH 256/420] Cleaned. --- api/service/explorer/service.go | 48 ------------------------------- cmd/harmony/main.go | 3 +- consensus/consensus_v2.go | 13 --------- consensus/validator.go | 11 ------- consensus/view_change.go | 7 ----- node/node_newblock.go | 1 - test/configs/local-resharding.txt | 31 ++++++++++++++------ test/deploy.sh | 5 +--- 8 files changed, 24 insertions(+), 95 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index b13f73db61..b5ffa1b8b0 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -7,13 +7,11 @@ import ( "fmt" "net" "net/http" - "os" "path" "strconv" "time" "github.com/RoaringBitmap/roaring/roaring64" - "github.com/harmony-one/harmony/internal/common" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" ethCommon "github.com/ethereum/go-ethereum/common" @@ -121,9 +119,6 @@ func (s *Service) Run() *http.Server { s.router.Path("/addresses").Queries("size", "{[0-9]*?}", "prefix", "{[a-zA-Z0-9]*?}").HandlerFunc(s.GetAddresses).Methods("GET") s.router.Path("/addresses").HandlerFunc(s.GetAddresses) s.router.Path("/height").HandlerFunc(s.GetHeight) - s.router.Path("/leader").HandlerFunc(s.GetLeader) - s.router.Path("/blocks").HandlerFunc(s.GetBlocks) - s.router.Path("/halt").HandlerFunc(s.halt) // Set up router for supply info s.router.Path("/burn-addresses").Queries().HandlerFunc(s.GetInaccessibleAddressInfo).Methods("GET") @@ -191,49 +186,6 @@ type HeightResponse struct { S3 uint64 `json:"3,omitempty"` } -func (s *Service) GetLeader(w http.ResponseWriter, r *http.Request) { - if s.backend.IsCurrentlyLeader() { - w.Write([]byte("true ")) - } else { - w.Write([]byte("false")) - } - - keys := "" - for _, p := range s.backend.GetPublicKeys() { - addr := common.Address{} - addrBytes := p.Object.GetAddress() - addr.SetBytes(addrBytes[:]) - keys += fmt.Sprintf("%s ", addr.String()) - break - } - //blsPubKeyBytes := leaderKey.Object.GetAddress() - //coinbase.SetBytes(blsPubKeyBytes[:]) - - w.Write([]byte(fmt.Sprintf(" %d", s.blockchain.ShardID()))) - w.Write([]byte(fmt.Sprintf(" %s", s.Port))) - w.Write([]byte(fmt.Sprintf(" %s", keys))) - w.Write([]byte(fmt.Sprintf(" %s", s.backend.GetPublicKeys().SerializeToHexStr()))) - -} - -func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { - cur := s.blockchain.CurrentHeader().Number().Uint64() - - for i := cur; i > 0; i-- { - block := s.blockchain.GetBlockByNumber(i) - leaderPubKey, _ := chain.GetLeaderPubKeyFromCoinbase(s.backend.Blockchain(), block.Header()) - w.Write([]byte(fmt.Sprintf("%s ", leaderPubKey.Bytes.Hex()))) - w.Write([]byte(fmt.Sprintf("#%d ", i))) - w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) - w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) - w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) - } -} - -func (s *Service) halt(w http.ResponseWriter, r *http.Request) { - os.Exit(0) -} - // GetHeight returns heights of current and beacon chains if needed. func (s *Service) GetHeight(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 4bb70a81b4..eacc392f10 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -425,9 +425,8 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { if currentNode.NodeConfig.Role() == nodeconfig.Validator { currentNode.RegisterValidatorServices() } else if currentNode.NodeConfig.Role() == nodeconfig.ExplorerNode { - + currentNode.RegisterExplorerServices() } - currentNode.RegisterExplorerServices() currentNode.RegisterService(service.CrosslinkSending, crosslink_sending.New(currentNode, currentNode.Blockchain())) if hc.Pprof.Enabled { setupPprofService(currentNode, hc) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index e8fa0fb17c..19d0c88907 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -678,8 +678,6 @@ func (consensus *Consensus) tryCatchup() error { } func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMessage) error { - // this function evaluates for all, leader and validators. - if consensus.Blockchain().CurrentBlock().NumberU64() < blk.NumberU64() { if _, err := consensus.Blockchain().InsertChain([]*types.Block{blk}, !consensus.FBFTLog.IsBlockVerified(blk.Hash())); err != nil { consensus.getLogger().Error().Err(err).Msg("[commitBlock] Failed to add block to chain") @@ -777,17 +775,6 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg consensus.resetState() } -func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, error) { - if epoch.Uint64() == 0 { - return 0, nil - } - epochBlock := consensus.Blockchain.GetBlockByNumber(shard.Schedule.EpochLastBlock(epoch.Uint64() - 1)) - if epochBlock == nil { - return 0, errors.Errorf("block not found for number %d", epoch.Uint64()-1) - } - return epochBlock.Header().ViewID().Uint64() + 1, nil -} - func (consensus *Consensus) postCatchup(initBN uint64) { if initBN < consensus.BlockNum() { consensus.getLogger().Info(). diff --git a/consensus/validator.go b/consensus/validator.go index 2b18003212..06aa9bc08c 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -18,7 +18,6 @@ import ( ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { - recvMsg, err := consensus.parseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). @@ -27,9 +26,6 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Msg("[OnAnnounce] Unparseable leader message") return } - if consensus.ShardID == 0 { - //fmt.Println("onAnnounce called ", recvMsg.BlockNum) - } // NOTE let it handle its own logs if !consensus.onAnnounceSanityChecks(recvMsg) { @@ -398,13 +394,6 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { consensus.getLogger().Info().Msg("[OnCommitted] Start consensus timer (new block added)") consensus.consensusTimeout[timeoutConsensus].Start() } - - //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum) - if blk != nil { - //consensus.ReshardingNextLeader(blk) - } else { - //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum, "blk is nil") - } } // Collect private keys that are part of the current committee. diff --git a/consensus/view_change.go b/consensus/view_change.go index 377a7098f7..06640c123d 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -367,13 +367,6 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { return } - consensus.getLogger().Debug(). - Err(err). - Interface("SenderPubkeys", recvMsg.SenderPubkeys). - Str("NextLeader", recvMsg.LeaderPubkey.Bytes.Hex()). - Str("myBLSPubKey", consensus.priKey.GetPublicKeys().SerializeToHexStr()). - Msg("[onViewChange] I am the Leader") - if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) { consensus.getLogger().Info(). Int64("have", consensus.Decider.SignersCount(quorum.ViewChange)). diff --git a/node/node_newblock.go b/node/node_newblock.go index f96477413f..5050e4d6a7 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -2,7 +2,6 @@ package node import ( "errors" - "fmt" "sort" "strings" "time" diff --git a/test/configs/local-resharding.txt b/test/configs/local-resharding.txt index fc3d3e97b4..4ce91ece1c 100644 --- a/test/configs/local-resharding.txt +++ b/test/configs/local-resharding.txt @@ -1,12 +1,25 @@ 127.0.0.1 9000 validator .hmy/65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204.key -127.0.0.1 9002 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key -127.0.0.1 9004 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key -127.0.0.1 9006 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key -127.0.0.1 9008 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key -127.0.0.1 9010 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key -127.0.0.1 9099 explorer null 0 +127.0.0.1 9002 validator .hmy/40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809.key +127.0.0.1 9004 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key +127.0.0.1 9006 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key +127.0.0.1 9008 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key +127.0.0.1 9010 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key +127.0.0.1 9012 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key +127.0.0.1 9014 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key +127.0.0.1 9016 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key +127.0.0.1 9018 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key +127.0.0.1 9020 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key +127.0.0.1 9022 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key +127.0.0.1 9200 explorer null 0 127.0.0.1 9100 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key -127.0.0.1 9102 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key -127.0.0.1 9104 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key -127.0.0.1 9106 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key \ No newline at end of file +127.0.0.1 9102 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key +127.0.0.1 9104 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key +127.0.0.1 9106 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key +127.0.0.1 9108 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key +127.0.0.1 9110 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key +127.0.0.1 9112 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key +127.0.0.1 9114 validator .hmy/f47238daef97d60deedbde5302d05dea5de67608f11f406576e363661f7dcbc4a1385948549b31a6c70f6fde8a391486.key +127.0.0.1 9116 validator .hmy/fc4b9c535ee91f015efff3f32fbb9d32cdd9bfc8a837bb3eee89b8fff653c7af2050a4e147ebe5c7233dc2d5df06ee0a.key +127.0.0.1 9118 validator .hmy/ca86e551ee42adaaa6477322d7db869d3e203c00d7b86c82ebee629ad79cb6d57b8f3db28336778ec2180e56a8e07296.key +127.0.0.1 9300 explorer null 1 \ No newline at end of file diff --git a/test/deploy.sh b/test/deploy.sh index fe22de57f0..9ab6943110 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -69,7 +69,7 @@ function launch_localnet() { if ${VERBOSE}; then verbosity=5 else - verbosity=5 + verbosity=3 fi base_args=(--log_folder "${log_folder}" --min_peers "${MIN}" --bootnodes "${BN_MA}" "--network_type=$NETWORK" --blspass file:"${ROOT}/.hmy/blspass.txt" "--dns=false" "--verbosity=${verbosity}" "--p2p.security.max-conn-per-ip=100") @@ -80,11 +80,8 @@ function launch_localnet() { while IFS='' read -r line || [[ -n "$line" ]]; do i=$((i + 1)) - - # Read config for i-th node form config file IFS=' ' read -r ip port mode bls_key shard node_config <<<"${line}" - echo "LINE: ${line} ${shard}" args=("${base_args[@]}" --ip "${ip}" --port "${port}" --key "/tmp/${ip}-${port}.key" --db_dir "${ROOT}/db-${ip}-${port}" "--broadcast_invalid_tx=false") if [[ -z "$ip" || -z "$port" ]]; then echo "skip empty node" From c52380f191e99155aeae9f6cabcde48df2f95c30 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 19 Sep 2022 21:40:21 +0700 Subject: [PATCH 257/420] Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. --- consensus/consensus_v2.go | 1 - 1 file changed, 1 deletion(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 19d0c88907..1fcb6e2e4b 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -11,7 +11,6 @@ import ( "github.com/ethereum/go-ethereum/common" bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" - "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" From 455b8e60eff7f56bcdd968ab59bc6710ba780063 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 17 Oct 2022 17:09:04 +0700 Subject: [PATCH 258/420] comment activation --- internal/params/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/params/config.go b/internal/params/config.go index 310172dc71..74f32483b3 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -110,7 +110,7 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(2), AllowlistEpoch: big.NewInt(2), LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 64, + LeaderRotationBlocksCount: 64, FeeCollectEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. From 692b7a2257dfa434921151a360d48b14b500527a Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 18 Oct 2022 12:17:17 +0700 Subject: [PATCH 259/420] 295 epoch --- internal/params/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/params/config.go b/internal/params/config.go index 74f32483b3..310172dc71 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -110,7 +110,7 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(2), AllowlistEpoch: big.NewInt(2), LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 64, + LeaderRotationBlocksCount: 64, FeeCollectEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. From 3ba619bcd36e80db9597ee180fe68b8d049874f1 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 8 Nov 2022 20:19:37 +0700 Subject: [PATCH 260/420] Fix failed tests. --- internal/params/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/params/config.go b/internal/params/config.go index 310172dc71..6168dc9bbd 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -315,8 +315,8 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } @@ -360,7 +360,7 @@ var ( big.NewInt(1), // LeaderRotationEpoch 64, // LeaderRotationBlocksCount - big.NewInt(0), // FeeCollectEpoch + big.NewInt(0), // FeeCollectEpoch } // TestRules ... From ca20ab73ecd263c5031bf9ad02b03c6424bbb994 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 9 Nov 2022 15:36:28 +0700 Subject: [PATCH 261/420] Fixed code review. --- consensus/consensus_v2.go | 3 +-- hmy/hmy.go | 2 -- internal/params/config.go | 9 ++++----- node/api.go | 6 ------ 4 files changed, 5 insertions(+), 15 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 1fcb6e2e4b..e6e6445914 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -137,7 +137,6 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb } func (consensus *Consensus) finalCommit() { - // THIS IS NOT GOOD PLACE FOR LEADER SWITCHING numCommits := consensus.Decider.SignersCount(quorum.Commit) consensus.getLogger().Info(). @@ -714,7 +713,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { if header == nil { return } - // Previous epoch, we should not change leader. + // Previous block was epoch block, we should not change leader. if header.Epoch().Uint64() != epoch.Uint64() { return } diff --git a/hmy/hmy.go b/hmy/hmy.go index 8089f04ba8..58e8d59edf 100644 --- a/hmy/hmy.go +++ b/hmy/hmy.go @@ -18,7 +18,6 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" - "github.com/harmony-one/harmony/multibls" commonRPC "github.com/harmony-one/harmony/rpc/common" "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" @@ -97,7 +96,6 @@ type NodeAPI interface { GetStakingTransactionsCount(address, txType string) (uint64, error) GetTraceResultByHash(hash common.Hash) (json.RawMessage, error) IsCurrentlyLeader() bool - GetPublicKeys() multibls.PublicKeys IsOutOfSync(shardID uint32) bool SyncStatus(shardID uint32) (bool, uint64, uint64) SyncPeers() map[string]int diff --git a/internal/params/config.go b/internal/params/config.go index 6168dc9bbd..725704bd06 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -274,7 +274,7 @@ var ( LeaderRotationEpoch: big.NewInt(4), LeaderRotationBlocksCount: 5, FeeCollectEpoch: big.NewInt(5), - LeaderRotationEpoch: big.NewInt(4), + LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 5, } @@ -357,10 +357,9 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount - big.NewInt(0), // FeeCollectEpoch + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount + big.NewInt(0), // FeeCollectEpoch } // TestRules ... diff --git a/node/api.go b/node/api.go index debcb201f6..ceda968084 100644 --- a/node/api.go +++ b/node/api.go @@ -6,7 +6,6 @@ import ( "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy" "github.com/harmony-one/harmony/internal/tikv" - "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/rosetta" hmy_rpc "github.com/harmony-one/harmony/rpc" rpc_common "github.com/harmony-one/harmony/rpc/common" @@ -19,11 +18,6 @@ func (node *Node) IsCurrentlyLeader() bool { return node.Consensus.IsLeader() } -// GetPublicKeys exposes if node is currently the leader node -func (node *Node) GetPublicKeys() multibls.PublicKeys { - return node.Consensus.GetPrivateKeys().GetPublicKeys() -} - // PeerConnectivity .. func (node *Node) PeerConnectivity() (int, int, int) { return node.host.PeerConnectivity() From 25fe6a2f6c471c7f2b6760bfd1e34c7594e4feeb Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 10 Nov 2022 12:57:57 +0700 Subject: [PATCH 262/420] Fix review comments. --- consensus/consensus.go | 5 +++++ consensus/consensus_v2.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 4cf4fdf6f8..1c9430caa5 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -184,6 +184,11 @@ func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper { func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } + +func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { + return consensus.LeaderPubKey +} + func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { consensus.pubKeyLock.Lock() consensus.LeaderPubKey = pub diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index e6e6445914..e9cdfe5ef4 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -713,7 +713,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { if header == nil { return } - // Previous block was epoch block, we should not change leader. + // Previous epoch, we should not change leader. if header.Epoch().Uint64() != epoch.Uint64() { return } From 5f17855f9224e60164ca1c8dd100c7cca8eea94b Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 17 Jan 2023 13:22:30 -0300 Subject: [PATCH 263/420] Merged leader rotation. --- api/service/stagedsync/stagedsync.go | 26 ++++++++++++-------------- consensus/consensus_service.go | 12 ------------ consensus/consensus_v2.go | 15 +++++---------- internal/params/config.go | 2 -- internal/utils/singleton.go | 2 -- 5 files changed, 17 insertions(+), 40 deletions(-) diff --git a/api/service/stagedsync/stagedsync.go b/api/service/stagedsync/stagedsync.go index aca707f619..83af6abf9f 100644 --- a/api/service/stagedsync/stagedsync.go +++ b/api/service/stagedsync/stagedsync.go @@ -1205,22 +1205,20 @@ func (ss *StagedSync) GetMaxPeerHeight() (uint64, error) { return ss.getMaxPeerHeight() } -func (ss *StagedSync) addConsensusLastMile(bc core.BlockChain, consensus *consensus.Consensus) error { +func (ss *StagedSync) addConsensusLastMile(bc core.BlockChain, cs *consensus.Consensus) error { curNumber := bc.CurrentBlock().NumberU64() - blockIter, err := consensus.GetLastMileBlockIter(curNumber + 1) - if err != nil { - return err - } - for { - block := blockIter.Next() - if block == nil { - break - } - if _, err := bc.InsertChain(types.Blocks{block}, true); err != nil { - return errors.Wrap(err, "failed to InsertChain") + return cs.GetLastMileBlockIter(curNumber+1, func(blockIter *consensus.LastMileBlockIter) error { + for { + block := blockIter.Next() + if block == nil { + break + } + if _, err := bc.InsertChain(types.Blocks{block}, true); err != nil { + return errors.Wrap(err, "failed to InsertChain") + } } - } - return nil + return nil + }) } // GetSyncingPort returns the syncing port. diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 411b9f6552..e62b2ed048 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -471,18 +471,6 @@ func (consensus *Consensus) IsLeader() bool { return consensus.isLeader() } -// IsLeader check if the node is a leader or not by comparing the public key of -// the node with the leader public key -func (consensus *Consensus) isLeader() bool { - obj := consensus.LeaderPubKey.Object - for _, key := range consensus.priKey { - if key.Pub.Object.IsEqual(obj) { - return true - } - } - return false -} - // isLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key. This function assume it runs under lock. func (consensus *Consensus) isLeader() bool { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index e9cdfe5ef4..3ef0ea0655 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -412,16 +412,6 @@ func (consensus *Consensus) tick() { } } -// Close closes the consensus. If current is in normal commit phase, wait until the commit -// phase end. -func (consensus *Consensus) Close() error { - if consensus.dHelper != nil { - consensus.dHelper.close() - } - consensus.waitForCommit() - return nil -} - func (consensus *Consensus) BlockChannel(newBlock *types.Block) { //consensus.ReshardingNextLeader(newBlock) consensus.getLogger().Info(). @@ -489,6 +479,11 @@ func (consensus *Consensus) GetLastMileBlockIter(bnStart uint64, cb func(iter *L consensus.mutex.Lock() defer consensus.mutex.Unlock() + return consensus.getLastMileBlockIter(bnStart, cb) +} + +// GetLastMileBlockIter get the iterator of the last mile blocks starting from number bnStart +func (consensus *Consensus) getLastMileBlockIter(bnStart uint64, cb func(iter *LastMileBlockIter) error) error { if consensus.BlockVerifier == nil { return errors.New("consensus haven't initialized yet") } diff --git a/internal/params/config.go b/internal/params/config.go index 725704bd06..f5ce665b1f 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -274,8 +274,6 @@ var ( LeaderRotationEpoch: big.NewInt(4), LeaderRotationBlocksCount: 5, FeeCollectEpoch: big.NewInt(5), - LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 5, } // AllProtocolChanges ... diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index 592e8dc6b3..10101d7673 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -6,8 +6,6 @@ import ( "fmt" "os" "path" - "runtime/debug" - "strconv" "strconv" "sync" "time" From 25c66d2f1530857c1632e667cf1399aa77d48ebf Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sat, 21 Jan 2023 17:07:40 -0300 Subject: [PATCH 264/420] Rebased on dev. --- api/service/consensus/service.go | 2 +- consensus/consensus.go | 8 ++------ consensus/consensus_service.go | 16 +--------------- consensus/consensus_v2.go | 20 +++++++------------- consensus/view_change.go | 14 ++++++++++---- 5 files changed, 21 insertions(+), 39 deletions(-) diff --git a/api/service/consensus/service.go b/api/service/consensus/service.go index d85a072f2e..49a1da64fb 100644 --- a/api/service/consensus/service.go +++ b/api/service/consensus/service.go @@ -32,5 +32,5 @@ func (s *Service) Stop() error { utils.Logger().Info().Msg("Stopping consensus service.") close(s.stopChan) utils.Logger().Info().Msg("Consensus service stopped.") - return s.consensus.Close() + return nil } diff --git a/consensus/consensus.go b/consensus/consensus.go index 1c9430caa5..bbf3634be9 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -178,6 +178,8 @@ func (consensus *Consensus) getPublicKeys() multibls.PublicKeys { } func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() return consensus.getLeaderPubKey() } @@ -189,12 +191,6 @@ func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } -func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = pub - consensus.pubKeyLock.Unlock() -} - func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e62b2ed048..a92a71f7e4 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -111,7 +111,7 @@ func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.Publi // do not reset view change state if it is in view changing mode if !consensus.isViewChangingMode() { - consensus.ResetViewChangeState() + consensus.resetViewChangeState() } return consensus.Decider.ParticipantsCount() } @@ -501,19 +501,6 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } -// SetLeaderIndex set the leader index. -func (consensus *Consensus) SetLeaderIndex(f func(int) int) (current int) { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() - consensus.LeaderIndex = f(consensus.LeaderIndex) - return consensus.LeaderIndex -} - -func (consensus *Consensus) GetLeaderIndex() int { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() - return consensus.LeaderIndex -} // SetCurBlockViewID set the current view ID func (consensus *Consensus) setCurBlockViewID(viewID uint64) { consensus.current.SetCurBlockViewID(viewID) @@ -524,7 +511,6 @@ func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) } - // SetViewChangingID set the current view change ID func (consensus *Consensus) setViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 3ef0ea0655..23c51ba248 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -414,18 +414,18 @@ func (consensus *Consensus) tick() { func (consensus *Consensus) BlockChannel(newBlock *types.Block) { //consensus.ReshardingNextLeader(newBlock) - consensus.getLogger().Info(). - Uint64("MsgBlockNum", newBlock.NumberU64()). - Msg("[ConsensusMainLoop] Received Proposed New Block!") + consensus.getLogger().Info(). + Uint64("MsgBlockNum", newBlock.NumberU64()). + Msg("[ConsensusMainLoop] Received Proposed New Block!") if newBlock.NumberU64() < consensus.BlockNum() { consensus.getLogger().Warn().Uint64("newBlockNum", newBlock.NumberU64()). Msg("[ConsensusMainLoop] received old block, abort") return - } - // Sleep to wait for the full block time - consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") - time.AfterFunc(time.Until(consensus.NextBlockDue), func() { + } + // Sleep to wait for the full block time + consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") + time.AfterFunc(time.Until(consensus.NextBlockDue), func() { consensus.StartFinalityCount() consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -750,12 +750,6 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg } else { epoch = blk.Epoch() } - var epoch *big.Int - if blk.IsLastBlockInEpoch() { - epoch = new(big.Int).Add(blk.Epoch(), common.Big1) - } else { - epoch = blk.Epoch() - } if consensus.Blockchain.Config().IsLeaderRotation(epoch) { consensus.rotateLeader(epoch) } diff --git a/consensus/view_change.go b/consensus/view_change.go index 06640c123d..3ab3d2c9da 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -327,8 +327,8 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri consensus.current.SetMode(Normal) consensus.consensusTimeout[timeoutViewChange].Stop() - consensus.SetViewIDs(viewID) - consensus.ResetViewChangeState() + consensus.setViewIDs(viewID) + consensus.resetViewChangeState() consensus.consensusTimeout[timeoutConsensus].Start() consensus.getLogger().Info(). @@ -340,7 +340,6 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.resetState() } - fmt.Println("[startNewView]", newLeaderPriKey.Pub.Bytes.Hex()) consensus.setLeaderPubKey(newLeaderPriKey.Pub) return nil @@ -532,7 +531,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // newView message verified success, override my state consensus.setViewIDs(recvMsg.ViewID) consensus.LeaderPubKey = senderKey - consensus.ResetViewChangeState() + consensus.resetViewChangeState() consensus.msgSender.StopRetry(msg_pb.MessageType_VIEWCHANGE) @@ -553,6 +552,13 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // ResetViewChangeState resets the view change structure func (consensus *Consensus) ResetViewChangeState() { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + consensus.resetViewChangeState() +} + +// ResetViewChangeState resets the view change structure +func (consensus *Consensus) resetViewChangeState() { consensus.getLogger().Info(). Str("Phase", consensus.phase.String()). Msg("[ResetViewChangeState] Resetting view change state") From cd97b995068ecd6c106f7c5461ebb781e54ab6b1 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Fri, 27 Jan 2023 01:52:17 -0300 Subject: [PATCH 265/420] Rebased on dev. --- consensus/consensus.go | 12 ------------ consensus/consensus_v2.go | 2 +- internal/params/config.go | 4 ---- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index bbf3634be9..080b39b33e 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -187,18 +187,6 @@ func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } -func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { - return consensus.LeaderPubKey -} - -func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { - return consensus.LeaderPubKey -} - -func (consensus *Consensus) setLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { - consensus.LeaderPubKey = pub -} - func (consensus *Consensus) setLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { consensus.LeaderPubKey = pub } diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 23c51ba248..44c3ebbd21 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -750,7 +750,7 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg } else { epoch = blk.Epoch() } - if consensus.Blockchain.Config().IsLeaderRotation(epoch) { + if consensus.Blockchain().Config().IsLeaderRotation(epoch) { consensus.rotateLeader(epoch) } diff --git a/internal/params/config.go b/internal/params/config.go index f5ce665b1f..8ee199d22a 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -711,10 +711,6 @@ func (c *ChainConfig) IsLeaderRotation(epoch *big.Int) bool { return isForked(c.LeaderRotationEpoch, epoch) } -func (c *ChainConfig) IsLeaderRotation(epoch *big.Int) bool { - return isForked(c.LeaderRotationEpoch, epoch) -} - // IsFeeCollectEpoch determines whether Txn Fees will be collected into the community-managed account. func (c *ChainConfig) IsFeeCollectEpoch(epoch *big.Int) bool { return isForked(c.FeeCollectEpoch, epoch) From 9dabe148577bdf49a948b0168308e08edd6c0cc0 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Fri, 27 Jan 2023 19:14:09 -0300 Subject: [PATCH 266/420] Fix usage of private methods. --- consensus/checks.go | 5 +++-- consensus/consensus.go | 4 ++++ consensus/consensus_service.go | 13 +++++++++++-- consensus/consensus_v2.go | 5 +++-- consensus/debug.go | 14 ++++++++++++++ consensus/validator.go | 2 +- consensus/view_change.go | 12 ++++++------ consensus/view_change_msg.go | 10 +++++----- core/blockchain_impl.go | 15 +++++++++++++-- 9 files changed, 60 insertions(+), 20 deletions(-) diff --git a/consensus/checks.go b/consensus/checks.go index 9e095a45c4..e4c9a7084f 100644 --- a/consensus/checks.go +++ b/consensus/checks.go @@ -56,9 +56,10 @@ func (consensus *Consensus) senderKeySanityChecks(msg *msg_pb.Message, senderKey } func (consensus *Consensus) isRightBlockNumAndViewID(recvMsg *FBFTMessage) bool { - if recvMsg.ViewID != consensus.GetCurBlockViewID() || recvMsg.BlockNum != consensus.BlockNum() { + blockNum := consensus.getBlockNum() + if recvMsg.ViewID != consensus.getCurBlockViewID() || recvMsg.BlockNum != blockNum { consensus.getLogger().Debug(). - Uint64("blockNum", consensus.BlockNum()). + Uint64("blockNum", blockNum). Str("recvMsg", recvMsg.String()). Msg("BlockNum/viewID not match") return false diff --git a/consensus/consensus.go b/consensus/consensus.go index 080b39b33e..2dbfa52a0c 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -224,6 +224,10 @@ func (consensus *Consensus) BlockNum() uint64 { return atomic.LoadUint64(&consensus.blockNum) } +func (consensus *Consensus) getBlockNum() uint64 { + return atomic.LoadUint64(&consensus.blockNum) +} + // New create a new Consensus record func New( host p2p.Host, shard uint32, multiBLSPriKey multibls.PrivateKeys, diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index a92a71f7e4..c43c7fe127 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -486,6 +486,8 @@ func (consensus *Consensus) isLeader() bool { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() consensus.setViewIDs(height) } @@ -625,11 +627,18 @@ func (consensus *Consensus) NumSignaturesIncludedInBlock(block *types.Block) uin return count } +// GetLogger returns logger for consensus contexts added. +func (consensus *Consensus) GetLogger() *zerolog.Logger { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() + return consensus.getLogger() +} + // getLogger returns logger for consensus contexts added func (consensus *Consensus) getLogger() *zerolog.Logger { logger := utils.Logger().With(). - Uint64("myBlock", consensus.BlockNum()). - Uint64("myViewID", consensus.GetCurBlockViewID()). + Uint64("myBlock", consensus.blockNum). + Uint64("myViewID", consensus.getCurBlockViewID()). Str("phase", consensus.phase.String()). Str("mode", consensus.current.Mode().String()). Logger() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 44c3ebbd21..1ab33d0bbf 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -46,6 +46,8 @@ const ( // IsViewChangingMode return true if curernt mode is viewchanging func (consensus *Consensus) IsViewChangingMode() bool { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() return consensus.isViewChangingMode() } @@ -413,8 +415,7 @@ func (consensus *Consensus) tick() { } func (consensus *Consensus) BlockChannel(newBlock *types.Block) { - //consensus.ReshardingNextLeader(newBlock) - consensus.getLogger().Info(). + consensus.GetLogger().Info(). Uint64("MsgBlockNum", newBlock.NumberU64()). Msg("[ConsensusMainLoop] Received Proposed New Block!") diff --git a/consensus/debug.go b/consensus/debug.go index ae1d2d92bf..da3587710a 100644 --- a/consensus/debug.go +++ b/consensus/debug.go @@ -12,10 +12,24 @@ func (consensus *Consensus) GetConsensusMode() string { // GetCurBlockViewID returns the current view ID of the consensus func (consensus *Consensus) GetCurBlockViewID() uint64 { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() + return consensus.getCurBlockViewID() +} + +// GetCurBlockViewID returns the current view ID of the consensus +func (consensus *Consensus) getCurBlockViewID() uint64 { return consensus.current.GetCurBlockViewID() } // GetViewChangingID returns the current view changing ID of the consensus func (consensus *Consensus) GetViewChangingID() uint64 { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() + return consensus.current.GetViewChangingID() +} + +// GetViewChangingID returns the current view changing ID of the consensus +func (consensus *Consensus) getViewChangingID() uint64 { return consensus.current.GetViewChangingID() } diff --git a/consensus/validator.go b/consensus/validator.go index 06aa9bc08c..f85cb8e3d8 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -277,7 +277,7 @@ func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { if committedMsg != nil { consensus.onCommitted(committedMsg) } - if curBlockNum < consensus.BlockNum() { + if curBlockNum < consensus.getBlockNum() { consensus.getLogger().Info().Msg("[OnPrepared] Successfully caught up with committed message") break } diff --git a/consensus/view_change.go b/consensus/view_change.go index 3ab3d2c9da..13acb86ae2 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -253,7 +253,7 @@ func (consensus *Consensus) startViewChange() { consensus.getLogger().Warn(). Uint64("nextViewID", nextViewID). - Uint64("viewChangingID", consensus.GetViewChangingID()). + Uint64("viewChangingID", consensus.getViewChangingID()). Dur("timeoutDuration", duration). Str("NextLeader", consensus.LeaderPubKey.Bytes.Hex()). Msg("[startViewChange]") @@ -270,7 +270,7 @@ func (consensus *Consensus) startViewChange() { if err := consensus.vc.InitPayload( consensus.FBFTLog, nextViewID, - consensus.BlockNum(), + consensus.getBlockNum(), consensus.priKey, members); err != nil { consensus.getLogger().Error().Err(err).Msg("[startViewChange] Init Payload Error") @@ -284,7 +284,7 @@ func (consensus *Consensus) startViewChange() { } msgToSend := consensus.constructViewChangeMessage(&key) if err := consensus.msgSender.SendWithRetry( - consensus.BlockNum(), + consensus.getBlockNum(), msg_pb.MessageType_VIEWCHANGE, []nodeconfig.GroupID{ nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, @@ -310,7 +310,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri } if err := consensus.msgSender.SendWithRetry( - consensus.BlockNum(), + consensus.getBlockNum(), msg_pb.MessageType_NEWVIEW, []nodeconfig.GroupID{ nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, @@ -450,10 +450,10 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { Msg("[onNewView] Received NewView Message") // change view and leaderKey to keep in sync with network - if consensus.BlockNum() != recvMsg.BlockNum { + if consensus.getBlockNum() != recvMsg.BlockNum { consensus.getLogger().Warn(). Uint64("MsgBlockNum", recvMsg.BlockNum). - Uint64("myBlockNum", consensus.BlockNum()). + Uint64("myBlockNum", consensus.getBlockNum()). Msg("[onNewView] Invalid block number") return } diff --git a/consensus/view_change_msg.go b/consensus/view_change_msg.go index 2433abdcbe..c241450306 100644 --- a/consensus/view_change_msg.go +++ b/consensus/view_change_msg.go @@ -23,8 +23,8 @@ func (consensus *Consensus) constructViewChangeMessage(priKey *bls.PrivateKeyWra Type: msg_pb.MessageType_VIEWCHANGE, Request: &msg_pb.Message_Viewchange{ Viewchange: &msg_pb.ViewChangeRequest{ - ViewId: consensus.GetViewChangingID(), - BlockNum: consensus.BlockNum(), + ViewId: consensus.getViewChangingID(), + BlockNum: consensus.getBlockNum(), ShardId: consensus.ShardID, SenderPubkey: priKey.Pub.Bytes[:], LeaderPubkey: consensus.LeaderPubKey.Bytes[:], @@ -33,7 +33,7 @@ func (consensus *Consensus) constructViewChangeMessage(priKey *bls.PrivateKeyWra } preparedMsgs := consensus.FBFTLog.GetMessagesByTypeSeq( - msg_pb.MessageType_PREPARED, consensus.BlockNum(), + msg_pb.MessageType_PREPARED, consensus.getBlockNum(), ) preparedMsg := consensus.FBFTLog.FindMessageByMaxViewID(preparedMsgs) @@ -83,7 +83,7 @@ func (consensus *Consensus) constructViewChangeMessage(priKey *bls.PrivateKeyWra } viewIDBytes := make([]byte, 8) - binary.LittleEndian.PutUint64(viewIDBytes, consensus.GetViewChangingID()) + binary.LittleEndian.PutUint64(viewIDBytes, consensus.getViewChangingID()) sign1 := priKey.Pri.SignHash(viewIDBytes) if sign1 != nil { vcMsg.ViewidSig = sign1.Serialize() @@ -107,7 +107,7 @@ func (consensus *Consensus) constructNewViewMessage(viewID uint64, priKey *bls.P Request: &msg_pb.Message_Viewchange{ Viewchange: &msg_pb.ViewChangeRequest{ ViewId: viewID, - BlockNum: consensus.BlockNum(), + BlockNum: consensus.getBlockNum(), ShardId: consensus.ShardID, SenderPubkey: priKey.Pub.Bytes[:], }, diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 783a571e89..8b0683bd65 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -327,8 +327,19 @@ func VerifyBlockCrossLinks(blockchain BlockChain, block *types.Block) error { // ReadCrossLink beacon chain usage. cl, err := blockchain.ReadCrossLink(crossLink.ShardID(), crossLink.BlockNum()) if err == nil && cl != nil { - // Add slash for exist same blocknum but different crosslink - return errAlreadyExist + utils.Logger().Err(errAlreadyExist). + Uint64("beacon-block-number", block.NumberU64()). + Interface("remote", crossLink). + Interface("local", cl). + Msg("[CrossLinkVerification]") + // TODO Add slash for exist same blocknum but different crosslink + return errors.Wrapf( + errAlreadyExist, + "[CrossLinkVerification] shard: %d block: %d on beacon block %d", + crossLink.ShardID(), + crossLink.BlockNum(), + block.NumberU64(), + ) } if err := VerifyCrossLink(blockchain, crossLink); err != nil { return errors.Wrapf(err, "cannot VerifyBlockCrossLinks") From 60fc56d1f3caae3342072fc825c0181281ec3049 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sat, 28 Jan 2023 00:21:05 -0300 Subject: [PATCH 267/420] Fix usage of private methods. --- consensus/checks.go | 2 +- consensus/consensus_service.go | 12 ++++++++++++ consensus/consensus_v2.go | 16 ++++++++-------- consensus/construct.go | 4 ++-- consensus/debug.go | 7 +++++++ consensus/leader.go | 8 ++++---- consensus/view_change.go | 12 ++++++------ 7 files changed, 40 insertions(+), 21 deletions(-) diff --git a/consensus/checks.go b/consensus/checks.go index e4c9a7084f..fadcfbd04b 100644 --- a/consensus/checks.go +++ b/consensus/checks.go @@ -147,7 +147,7 @@ func (consensus *Consensus) onViewChangeSanityCheck(recvMsg *FBFTMessage) bool { consensus.getLogger().Debug(). Uint64("MsgBlockNum", recvMsg.BlockNum). - Uint64("MyViewChangingID", consensus.GetViewChangingID()). + Uint64("MyViewChangingID", consensus.getViewChangingID()). Uint64("MsgViewChangingID", recvMsg.ViewID). Interface("SendPubKeys", recvMsg.SenderPubkeys). Msg("[onViewChangeSanityCheck]") diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index c43c7fe127..1ce41392cc 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -210,6 +210,13 @@ func (consensus *Consensus) SetIsBackup(isBackup bool) { // Mode returns the mode of consensus func (consensus *Consensus) Mode() Mode { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() + return consensus.mode() +} + +// mode returns the mode of consensus +func (consensus *Consensus) mode() Mode { return consensus.current.Mode() } @@ -254,6 +261,11 @@ func (consensus *Consensus) SetBlockNum(blockNum uint64) { atomic.StoreUint64(&consensus.blockNum, blockNum) } +// SetBlockNum sets the blockNum in consensus object, called at node bootstrap +func (consensus *Consensus) setBlockNum(blockNum uint64) { + atomic.StoreUint64(&consensus.blockNum, blockNum) +} + // ReadSignatureBitmapPayload read the payload for signature and bitmap; offset is the beginning position of reading func (consensus *Consensus) ReadSignatureBitmapPayload(recvPayload []byte, offset int) (*bls_core.Sign, *bls_cosi.Mask, error) { consensus.mutex.RLock() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 1ab33d0bbf..4244cb3732 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -367,7 +367,7 @@ func (consensus *Consensus) syncReadyChan() { func (consensus *Consensus) syncNotReadyChan() { consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncNotReadyChan") - consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) + consensus.setBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) consensus.current.SetMode(Syncing) consensus.getLogger().Info().Msg("[ConsensusMainLoop] Node is OUT OF SYNC") consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() @@ -452,14 +452,14 @@ func (consensus *Consensus) BlockChannel(newBlock *types.Block) { // waitForCommit wait extra 2 seconds for commit phase to finish func (consensus *Consensus) waitForCommit() { - if consensus.Mode() != Normal || consensus.phase.Get() != FBFTCommit { + if consensus.mode() != Normal || consensus.phase.Get() != FBFTCommit { return } // We only need to wait consensus is in normal commit phase utils.Logger().Warn().Str("phase", consensus.phase.String()).Msg("[shutdown] commit phase has to wait") maxWait := time.Now().Add(2 * consensus.BlockPeriod) - for time.Now().Before(maxWait) && consensus.GetConsensusPhase() == "Commit" { + for time.Now().Before(maxWait) && consensus.getConsensusPhase() == "Commit" { utils.Logger().Warn().Msg("[shutdown] wait for consensus finished") time.Sleep(time.Millisecond * 100) } @@ -635,7 +635,7 @@ func (consensus *Consensus) tryCatchup() error { if consensus.BlockVerifier == nil { return errors.New("consensus haven't finished initialization") } - initBN := consensus.BlockNum() + initBN := consensus.getBlockNum() defer consensus.postCatchup(initBN) blks, msgs, err := consensus.getLastMileBlocksAndMsg(initBN) @@ -764,15 +764,15 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg } func (consensus *Consensus) postCatchup(initBN uint64) { - if initBN < consensus.BlockNum() { + if initBN < consensus.getBlockNum() { consensus.getLogger().Info(). Uint64("From", initBN). - Uint64("To", consensus.BlockNum()). + Uint64("To", consensus.getBlockNum()). Msg("[TryCatchup] Caught up!") consensus.switchPhase("TryCatchup", FBFTAnnounce) } // catch up and skip from view change trap - if initBN < consensus.BlockNum() && consensus.isViewChangingMode() { + if initBN < consensus.getBlockNum() && consensus.isViewChangingMode() { consensus.current.SetMode(Normal) consensus.consensusTimeout[timeoutViewChange].Stop() } @@ -833,7 +833,7 @@ func (consensus *Consensus) GenerateVdfAndProof(newBlock *types.Block, vrfBlockN start := time.Now() vdf.Execute() duration := time.Since(start) - consensus.getLogger().Info(). + consensus.GetLogger().Info(). Dur("duration", duration). Msg("[ConsensusMainLoop] VDF computation finished") output := <-outputChannel diff --git a/consensus/construct.go b/consensus/construct.go index 8d9f3838aa..eaa6fd83f1 100644 --- a/consensus/construct.go +++ b/consensus/construct.go @@ -29,8 +29,8 @@ type NetworkMessage struct { func (consensus *Consensus) populateMessageFields( request *msg_pb.ConsensusRequest, blockHash []byte, ) *msg_pb.ConsensusRequest { - request.ViewId = consensus.GetCurBlockViewID() - request.BlockNum = consensus.BlockNum() + request.ViewId = consensus.getCurBlockViewID() + request.BlockNum = consensus.getBlockNum() request.ShardId = consensus.ShardID // 32 byte block hash request.BlockHash = blockHash diff --git a/consensus/debug.go b/consensus/debug.go index da3587710a..cba13cc011 100644 --- a/consensus/debug.go +++ b/consensus/debug.go @@ -2,6 +2,13 @@ package consensus // GetConsensusPhase returns the current phase of the consensus. func (consensus *Consensus) GetConsensusPhase() string { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() + return consensus.getConsensusPhase() +} + +// GetConsensusPhase returns the current phase of the consensus. +func (consensus *Consensus) getConsensusPhase() string { return consensus.phase.String() } diff --git a/consensus/leader.go b/consensus/leader.go index 548ec98c0a..2f7766e19b 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -74,7 +74,7 @@ func (consensus *Consensus) announce(block *types.Block) { } // Construct broadcast p2p message if err := consensus.msgSender.SendWithRetry( - consensus.BlockNum(), msg_pb.MessageType_ANNOUNCE, []nodeconfig.GroupID{ + consensus.getBlockNum(), msg_pb.MessageType_ANNOUNCE, []nodeconfig.GroupID{ nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)), }, p2p.ConstructMessage(msgToSend)); err != nil { consensus.getLogger().Warn(). @@ -95,7 +95,7 @@ func (consensus *Consensus) announce(block *types.Block) { func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { // TODO(audit): make FBFT lookup using map instead of looping through all items. if !consensus.FBFTLog.HasMatchingViewAnnounce( - consensus.BlockNum(), consensus.GetCurBlockViewID(), recvMsg.BlockHash, + consensus.getBlockNum(), consensus.getCurBlockViewID(), recvMsg.BlockHash, ) { consensus.getLogger().Debug(). Uint64("MsgViewID", recvMsg.ViewID). @@ -280,7 +280,7 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { //// Write - End //// Read - Start - viewID := consensus.GetCurBlockViewID() + viewID := consensus.getCurBlockViewID() if consensus.Decider.IsAllSigsCollected() { logger.Info().Msg("[OnCommit] 100% Enough commits received") @@ -315,7 +315,7 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.mutex.Lock() defer consensus.mutex.Unlock() - if viewID == consensus.GetCurBlockViewID() { + if viewID == consensus.getCurBlockViewID() { consensus.finalCommit() } }(viewID) diff --git a/consensus/view_change.go b/consensus/view_change.go index 13acb86ae2..d8b1058a04 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -88,14 +88,14 @@ func (pm *State) SetIsBackup(isBackup bool) { // fallbackNextViewID return the next view ID and duration when there is an exception // to calculate the time-based viewId func (consensus *Consensus) fallbackNextViewID() (uint64, time.Duration) { - diff := int64(consensus.GetViewChangingID() + 1 - consensus.GetCurBlockViewID()) + diff := int64(consensus.getViewChangingID() + 1 - consensus.getCurBlockViewID()) if diff <= 0 { diff = int64(1) } consensus.getLogger().Error(). Int64("diff", diff). Msg("[fallbackNextViewID] use legacy viewID algorithm") - return consensus.GetViewChangingID() + 1, time.Duration(diff * diff * int64(viewChangeDuration)) + return consensus.getViewChangingID() + 1, time.Duration(diff * diff * int64(viewChangeDuration)) } // getNextViewID return the next view ID based on the timestamp @@ -152,7 +152,7 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrapper { gap := 1 - cur := consensus.GetCurBlockViewID() + cur := consensus.getCurBlockViewID() if viewID > cur { gap = int(viewID - cur) } @@ -196,7 +196,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe Str("leaderPubKey", consensus.LeaderPubKey.Bytes.Hex()). Int("gap", gap). Uint64("newViewID", viewID). - Uint64("myCurBlockViewID", consensus.GetCurBlockViewID()). + Uint64("myCurBlockViewID", consensus.getCurBlockViewID()). Msg("[getNextLeaderKey] got leaderPubKey from coinbase") // wasFound, next := consensus.Decider.NthNext(lastLeaderPubKey, gap) // FIXME: rotate leader on harmony nodes only before fully externalization @@ -234,7 +234,7 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - if consensus.disableViewChange || consensus.IsBackup() { + if consensus.disableViewChange || consensus.isBackup { return } @@ -242,7 +242,7 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() - consensus.SetViewChangingID(nextViewID) + consensus.setViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet // we use it this way as in many code we validate the messages From ccff38e304b83e6f81edd22e97f8a39a47a452bf Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sat, 28 Jan 2023 01:11:28 -0300 Subject: [PATCH 268/420] Fix usage of private methods. --- consensus/checks.go | 12 ++++++------ consensus/consensus_service.go | 4 ++-- consensus/consensus_v2.go | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/consensus/checks.go b/consensus/checks.go index fadcfbd04b..28da66ad7b 100644 --- a/consensus/checks.go +++ b/consensus/checks.go @@ -152,7 +152,7 @@ func (consensus *Consensus) onViewChangeSanityCheck(recvMsg *FBFTMessage) bool { Interface("SendPubKeys", recvMsg.SenderPubkeys). Msg("[onViewChangeSanityCheck]") - if consensus.BlockNum() > recvMsg.BlockNum { + if consensus.getBlockNum() > recvMsg.BlockNum { consensus.getLogger().Debug(). Msg("[onViewChange] Message BlockNum Is Low") return false @@ -163,13 +163,13 @@ func (consensus *Consensus) onViewChangeSanityCheck(recvMsg *FBFTMessage) bool { return false } if consensus.isViewChangingMode() && - consensus.GetCurBlockViewID() > recvMsg.ViewID { - consensus.getLogger().Debug().Uint64("curBlockViewID", consensus.GetCurBlockViewID()). + consensus.getCurBlockViewID() > recvMsg.ViewID { + consensus.getLogger().Debug().Uint64("curBlockViewID", consensus.getCurBlockViewID()). Uint64("msgViewID", recvMsg.ViewID). Msg("[onViewChangeSanityCheck] ViewChanging ID Is Low") return false } - if recvMsg.ViewID > consensus.GetViewChangingID() && recvMsg.ViewID-consensus.GetViewChangingID() > MaxViewIDDiff { + if recvMsg.ViewID > consensus.getViewChangingID() && recvMsg.ViewID-consensus.getViewChangingID() > MaxViewIDDiff { consensus.getLogger().Debug(). Msg("[onViewChangeSanityCheck] Received viewID that is MaxViewIDDiff (249) further from the current viewID!") return false @@ -194,9 +194,9 @@ func (consensus *Consensus) onViewChangeSanityCheck(recvMsg *FBFTMessage) bool { // TODO: leo: move the sanity check to p2p message validation func (consensus *Consensus) onNewViewSanityCheck(recvMsg *FBFTMessage) bool { - if recvMsg.ViewID < consensus.GetCurBlockViewID() { + if recvMsg.ViewID < consensus.getCurBlockViewID() { consensus.getLogger().Warn(). - Uint64("LastSuccessfulConsensusViewID", consensus.GetCurBlockViewID()). + Uint64("LastSuccessfulConsensusViewID", consensus.getCurBlockViewID()). Uint64("MsgViewChangingID", recvMsg.ViewID). Msg("[onNewView] ViewID should be larger than the viewID of the last successful consensus") return false diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 1ce41392cc..71582df33a 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -248,9 +248,9 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { Str("leaderKey", consensus.LeaderPubKey.Bytes.Hex()). Msg("[checkViewID] Start consensus timer") return nil - } else if msg.ViewID > consensus.GetCurBlockViewID() { + } else if msg.ViewID > consensus.getCurBlockViewID() { return consensus_engine.ErrViewIDNotMatch - } else if msg.ViewID < consensus.GetCurBlockViewID() { + } else if msg.ViewID < consensus.getCurBlockViewID() { return errors.New("view ID belongs to the past") } return nil diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 4244cb3732..fe6b802b33 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -144,7 +144,7 @@ func (consensus *Consensus) finalCommit() { consensus.getLogger().Info(). Int64("NumCommits", numCommits). Msg("[finalCommit] Finalizing Consensus") - beforeCatchupNum := consensus.BlockNum() + beforeCatchupNum := consensus.getBlockNum() leaderPriKey, err := consensus.getConsensusLeaderPrivateKey() if err != nil { @@ -345,16 +345,16 @@ func (consensus *Consensus) StartChannel() { func (consensus *Consensus) syncReadyChan() { consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncReadyChan") - if consensus.BlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { - consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) - consensus.SetViewIDs(consensus.Blockchain().CurrentHeader().ViewID().Uint64() + 1) + if consensus.getBlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { + consensus.setBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) + consensus.setViewIDs(consensus.Blockchain().CurrentHeader().ViewID().Uint64() + 1) mode := consensus.updateConsensusInformation() consensus.current.SetMode(mode) consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") consensus.consensusTimeout[timeoutConsensus].Start() consensus.getLogger().Info().Str("Mode", mode.String()).Msg("Node is IN SYNC") consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() - } else if consensus.Mode() == Syncing { + } else if consensus.mode() == Syncing { // Corner case where sync is triggered before `onCommitted` and there is a race // for block insertion between consensus and downloader. mode := consensus.updateConsensusInformation() From 268ef801ade99dd62b8ded6b771649a523906fb9 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 31 Jan 2023 12:04:33 -0300 Subject: [PATCH 269/420] Removed deadcode, LockedFBFTPhase. --- consensus/consensus.go | 4 ++-- consensus/consensus_fbft.go | 23 ----------------------- consensus/consensus_fbft_test.go | 15 --------------- consensus/consensus_service.go | 2 +- consensus/consensus_v2.go | 15 --------------- 5 files changed, 3 insertions(+), 56 deletions(-) delete mode 100644 consensus/consensus_fbft.go delete mode 100644 consensus/consensus_fbft_test.go diff --git a/consensus/consensus.go b/consensus/consensus.go index 2dbfa52a0c..1d3353f514 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -47,7 +47,7 @@ type Consensus struct { // FBFTLog stores the pbft messages and blocks during FBFT process FBFTLog *FBFTLog // phase: different phase of FBFT protocol: pre-prepare, prepare, commit, finish etc - phase *LockedFBFTPhase + phase FBFTPhase // current indicates what state a node is in current State // isBackup declarative the node is in backup mode @@ -246,7 +246,7 @@ func New( consensus.BlockNumLowChan = make(chan struct{}, 1) // FBFT related consensus.FBFTLog = NewFBFTLog() - consensus.phase = NewLockedFBFTPhase(FBFTAnnounce) + consensus.phase = FBFTAnnounce consensus.current = State{mode: Normal} // FBFT timeout consensus.consensusTimeout = createTimeout() diff --git a/consensus/consensus_fbft.go b/consensus/consensus_fbft.go deleted file mode 100644 index 29c9714d35..0000000000 --- a/consensus/consensus_fbft.go +++ /dev/null @@ -1,23 +0,0 @@ -package consensus - -type LockedFBFTPhase struct { - phase FBFTPhase -} - -func NewLockedFBFTPhase(initialPhrase FBFTPhase) *LockedFBFTPhase { - return &LockedFBFTPhase{ - phase: initialPhrase, - } -} - -func (a *LockedFBFTPhase) Set(phrase FBFTPhase) { - a.phase = phrase -} - -func (a *LockedFBFTPhase) Get() FBFTPhase { - return a.phase -} - -func (a *LockedFBFTPhase) String() string { - return a.Get().String() -} diff --git a/consensus/consensus_fbft_test.go b/consensus/consensus_fbft_test.go deleted file mode 100644 index a84cc3c832..0000000000 --- a/consensus/consensus_fbft_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package consensus - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestLockedFBFTPhase(t *testing.T) { - s := NewLockedFBFTPhase(FBFTAnnounce) - require.Equal(t, FBFTAnnounce, s.Get()) - - s.Set(FBFTCommit) - require.Equal(t, FBFTCommit, s.Get()) -} diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 71582df33a..0e4bb68141 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -554,7 +554,7 @@ func (consensus *Consensus) switchPhase(subject string, desired FBFTPhase) { Str("to:", desired.String()). Str("switchPhase:", subject) - consensus.phase.Set(desired) + consensus.phase = desired } var ( diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index fe6b802b33..3f9a2cfa2d 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -450,21 +450,6 @@ func (consensus *Consensus) BlockChannel(newBlock *types.Block) { } } -// waitForCommit wait extra 2 seconds for commit phase to finish -func (consensus *Consensus) waitForCommit() { - if consensus.mode() != Normal || consensus.phase.Get() != FBFTCommit { - return - } - // We only need to wait consensus is in normal commit phase - utils.Logger().Warn().Str("phase", consensus.phase.String()).Msg("[shutdown] commit phase has to wait") - - maxWait := time.Now().Add(2 * consensus.BlockPeriod) - for time.Now().Before(maxWait) && consensus.getConsensusPhase() == "Commit" { - utils.Logger().Warn().Msg("[shutdown] wait for consensus finished") - time.Sleep(time.Millisecond * 100) - } -} - // LastMileBlockIter is the iterator to iterate over the last mile blocks in consensus cache. // All blocks returned are guaranteed to pass the verification. type LastMileBlockIter struct { From e9843f9872fad352b3c196cae682c9810690841f Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 31 Jan 2023 12:08:32 -0300 Subject: [PATCH 270/420] Fix review comment. --- consensus/consensus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 1d3353f514..ac15adc15f 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -154,8 +154,8 @@ func (consensus *Consensus) verifyBlock(block *types.Block) error { // thus the blockchain is likely to be up to date. func (consensus *Consensus) BlocksSynchronized() { consensus.mutex.Lock() + defer consensus.mutex.Unlock() consensus.syncReadyChan() - consensus.mutex.Unlock() } // BlocksNotSynchronized lets the main loop know that block is not synchronized From 1ac5737e3b36a440bc038e097048f6d31169d9f6 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 31 Jan 2023 12:12:43 -0300 Subject: [PATCH 271/420] Fix review comment. --- consensus/consensus_v2.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 3f9a2cfa2d..6b0d54648d 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -310,9 +310,7 @@ func (consensus *Consensus) Start( case <-stopChan: return case <-ticker.C: - consensus.mutex.Lock() - consensus.tick() - consensus.mutex.Unlock() + consensus.Tick() } } }() @@ -373,6 +371,12 @@ func (consensus *Consensus) syncNotReadyChan() { consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() } +func (consensus *Consensus) Tick() { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + consensus.tick() +} + func (consensus *Consensus) tick() { if !consensus.start && consensus.isInitialLeader { return From 92add1f795d263cd2f85f2ee65a6d68c2531a1a8 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 31 Jan 2023 14:29:17 -0300 Subject: [PATCH 272/420] Go mod tidy. --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f70708b562..4ebe35fa3b 100644 --- a/go.mod +++ b/go.mod @@ -71,6 +71,7 @@ require ( github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b github.com/ledgerwatch/erigon-lib v0.0.0-20221218022306-0f8fdd40c2db github.com/ledgerwatch/log/v3 v3.6.0 + github.com/libp2p/go-libp2p-core v0.20.1 ) require ( @@ -165,7 +166,6 @@ require ( github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect - github.com/libp2p/go-libp2p-core v0.20.1 // indirect github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect github.com/libp2p/go-msgio v0.2.0 // indirect From 567c522b3518f26a736914849faf375c4fd803ef Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 31 Jan 2023 18:03:38 -0300 Subject: [PATCH 273/420] Set to EpochTBD. --- consensus/consensus_test.go | 2 +- internal/params/config.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/consensus_test.go b/consensus/consensus_test.go index 7d74d8815a..41fe5b127f 100644 --- a/consensus/consensus_test.go +++ b/consensus/consensus_test.go @@ -39,7 +39,7 @@ func TestConsensusInitialization(t *testing.T) { // FBFTLog assert.Equal(t, fbtLog, consensus.FBFTLog) - assert.Equal(t, FBFTAnnounce, consensus.phase.Get()) + assert.Equal(t, FBFTAnnounce, consensus.phase) // State / consensus.current assert.Equal(t, state.mode, consensus.current.mode) diff --git a/internal/params/config.go b/internal/params/config.go index 8ee199d22a..b50aa35532 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -271,7 +271,7 @@ var ( SlotsLimitedEpoch: EpochTBD, // epoch to enable HIP-16 CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, - LeaderRotationEpoch: big.NewInt(4), + LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 5, FeeCollectEpoch: big.NewInt(5), } From e6e936e34763ddf3fbb0803098df81dfb4875918 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 31 Jan 2023 19:27:43 -0300 Subject: [PATCH 274/420] Fix tests. --- consensus/view_change_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/consensus/view_change_test.go b/consensus/view_change_test.go index fc80b6ccfe..2d149b6b7b 100644 --- a/consensus/view_change_test.go +++ b/consensus/view_change_test.go @@ -43,7 +43,7 @@ func TestPhaseSwitching(t *testing.T) { _, _, consensus, _, err := GenerateConsensusForTesting() assert.NoError(t, err) - assert.Equal(t, FBFTAnnounce, consensus.phase.Get()) // It's a new consensus, we should be at the FBFTAnnounce phase. + assert.Equal(t, FBFTAnnounce, consensus.phase) // It's a new consensus, we should be at the FBFTAnnounce phase. switches := []phaseSwitch{ {start: FBFTAnnounce, end: FBFTPrepare}, @@ -73,10 +73,10 @@ func TestPhaseSwitching(t *testing.T) { func testPhaseGroupSwitching(t *testing.T, consensus *Consensus, phases []FBFTPhase, startPhase FBFTPhase, desiredPhase FBFTPhase) { for range phases { consensus.switchPhase("test", desiredPhase) - assert.Equal(t, desiredPhase, consensus.phase.Get()) + assert.Equal(t, desiredPhase, consensus.phase) } - assert.Equal(t, desiredPhase, consensus.phase.Get()) + assert.Equal(t, desiredPhase, consensus.phase) return } From 8811714718104ba08a21076a5ec1ab9ae38ccd82 Mon Sep 17 00:00:00 2001 From: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Date: Thu, 16 Feb 2023 02:57:49 +0000 Subject: [PATCH 275/420] [core] fix state handling of self destruct If a contract self destructs to self and then receives funds within the same transaction, it is possible for its stale state to be saved. This change removes that possibility by checking for deleted state objects before returning them. --- core/state/statedb.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index d571d3372d..0a7cd3b564 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -542,7 +542,10 @@ func (db *DB) createObject(addr common.Address) (newobj, prev *Object) { db.journal.append(resetObjectChange{prev: prev}) } db.setStateObject(newobj) - return newobj, prev + if prev != nil && !prev.deleted { + return newobj, prev + } + return newobj, nil } // CreateAccount explicitly creates a state object. If a state object with the address From 2c119a6e73557913e13182fbfa5351b53df7bbac Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sun, 19 Feb 2023 15:43:04 +0000 Subject: [PATCH 276/420] Fixed race error. --- api/service/explorer/schema.go | 2 +- api/service/explorer/storage.go | 54 ++++++++++++++++++++++++++++++--- go.mod | 2 +- go.sum | 4 +-- scripts/go_executable_build.sh | 2 +- 5 files changed, 54 insertions(+), 10 deletions(-) diff --git a/api/service/explorer/schema.go b/api/service/explorer/schema.go index 09848573d0..3104d75dc5 100644 --- a/api/service/explorer/schema.go +++ b/api/service/explorer/schema.go @@ -41,7 +41,7 @@ func readCheckpointBitmap(db databaseReader) (*roaring64.Bitmap, error) { } // writeCheckpointBitmap write explorer checkpoint bitmap to storage -func writeCheckpointBitmap(db databaseWriter, rb *roaring64.Bitmap) error { +func writeCheckpointBitmap(db databaseWriter, rb Bitmap) error { bitmapByte, err := rb.MarshalBinary() if err != nil { return err diff --git a/api/service/explorer/storage.go b/api/service/explorer/storage.go index 72d0281805..ebb4e2577b 100644 --- a/api/service/explorer/storage.go +++ b/api/service/explorer/storage.go @@ -34,11 +34,54 @@ const ( // explorer db is doing migration and unavailable var ErrExplorerNotReady = errors.New("explorer db not ready") +type Bitmap interface { + Clone() *roaring64.Bitmap + MarshalBinary() ([]byte, error) + CheckedAdd(x uint64) bool + Contains(x uint64) bool +} + +type ThreadSafeBitmap struct { + b Bitmap + mu sync.Mutex +} + +func (a *ThreadSafeBitmap) Clone() *roaring64.Bitmap { + a.mu.Lock() + defer a.mu.Unlock() + return a.b.Clone() +} + +func (a *ThreadSafeBitmap) CheckedAdd(x uint64) bool { + a.mu.Lock() + defer a.mu.Unlock() + return a.b.CheckedAdd(x) +} + +func (a *ThreadSafeBitmap) Contains(x uint64) bool { + a.mu.Lock() + defer a.mu.Unlock() + return a.b.Contains(x) +} + +func (a *ThreadSafeBitmap) MarshalBinary() ([]byte, error) { + a.mu.Lock() + defer a.mu.Unlock() + return a.b.MarshalBinary() +} + +func NewThreadSafeBitmap(bitmap Bitmap) Bitmap { + return &ThreadSafeBitmap{ + b: bitmap, + mu: sync.Mutex{}, + } +} + type ( storage struct { db database bc core.BlockChain - rb *roaring64.Bitmap + rb Bitmap //*roaring64.Bitmap // TODO: optimize this with priority queue tm *taskManager @@ -89,11 +132,12 @@ func newStorage(hc *harmonyconfig.HarmonyConfig, bc core.BlockChain, dbPath stri return nil, err } + safeBitmap := NewThreadSafeBitmap(bitmap) return &storage{ db: db, bc: bc, - rb: bitmap, - tm: newTaskManager(bitmap), + rb: safeBitmap, + tm: newTaskManager(safeBitmap), resultC: make(chan blockResult, numWorker), resultT: make(chan *traceResult, numWorker), available: abool.New(), @@ -211,14 +255,14 @@ type taskManager struct { blocksLP []*types.Block // blocks with low priorities lock sync.Mutex - rb *roaring64.Bitmap + rb Bitmap rbChangedCount int C chan struct{} T chan *traceResult } -func newTaskManager(bitmap *roaring64.Bitmap) *taskManager { +func newTaskManager(bitmap Bitmap) *taskManager { return &taskManager{ rb: bitmap, C: make(chan struct{}, numWorker), diff --git a/go.mod b/go.mod index 4ebe35fa3b..18e16a0c05 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/harmony-one/harmony go 1.19 require ( - github.com/RoaringBitmap/roaring v1.2.1 + github.com/RoaringBitmap/roaring v1.2.3 github.com/VictoriaMetrics/fastcache v1.5.7 github.com/Workiva/go-datastructures v1.0.50 github.com/allegro/bigcache v1.2.1 diff --git a/go.sum b/go.sum index 19d3d9e3da..2ebb5e94a2 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us= github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= -github.com/RoaringBitmap/roaring v1.2.1 h1:58/LJlg/81wfEHd5L9qsHduznOIhyv4qb1yWcSvVq9A= -github.com/RoaringBitmap/roaring v1.2.1/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= +github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= +github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw= diff --git a/scripts/go_executable_build.sh b/scripts/go_executable_build.sh index 2c1cc99f0e..6331aa5225 100755 --- a/scripts/go_executable_build.sh +++ b/scripts/go_executable_build.sh @@ -21,7 +21,7 @@ VERBOSE= if [ "$(uname -s)" == "Darwin" ]; then GO_GCFLAGS="" else - GO_GCFLAGS="all=-c 2" + GO_GCFLAGS="" fi DEBUG=false STATIC=true From bc3aec4ef6b9a093db90e3189be7d126bcc74877 Mon Sep 17 00:00:00 2001 From: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Date: Thu, 23 Feb 2023 06:19:04 +0000 Subject: [PATCH 277/420] rpc: add configurable http and `eth_call` timeout --- cmd/harmony/config_migrations.go | 17 +++++ cmd/harmony/default.go | 6 +- cmd/harmony/flags.go | 39 +++++++++++- cmd/harmony/flags_test.go | 58 +++++++++++++++++ internal/configs/harmony/harmony.go | 64 +++++++++++++++++++ internal/configs/harmony/harmony_test.go | 81 ++++++++++++++++++++++++ internal/configs/node/config.go | 7 ++ internal/configs/node/network.go | 6 ++ rosetta/rosetta.go | 5 +- rosetta/services/call_service.go | 10 ++- rpc/contract.go | 22 +++++-- rpc/rpc.go | 25 +++++--- 12 files changed, 320 insertions(+), 20 deletions(-) create mode 100644 internal/configs/harmony/harmony_test.go diff --git a/cmd/harmony/config_migrations.go b/cmd/harmony/config_migrations.go index 7c9420a0be..b3da9ec2ba 100644 --- a/cmd/harmony/config_migrations.go +++ b/cmd/harmony/config_migrations.go @@ -340,6 +340,23 @@ func init() { return confTree } + migrations["2.5.12"] = func(confTree *toml.Tree) *toml.Tree { + if confTree.Get("HTTP.ReadTimeout") == nil { + confTree.Set("HTTP.ReadTimeout", defaultConfig.HTTP.ReadTimeout) + } + if confTree.Get("HTTP.WriteTimeout") == nil { + confTree.Set("HTTP.WriteTimeout", defaultConfig.HTTP.WriteTimeout) + } + if confTree.Get("HTTP.IdleTimeout") == nil { + confTree.Set("HTTP.IdleTimeout", defaultConfig.HTTP.IdleTimeout) + } + if confTree.Get("RPCOpt.EvmCallTimeout") == nil { + confTree.Set("RPCOpt.EvmCallTimeout", defaultConfig.RPCOpt.EvmCallTimeout) + } + confTree.Set("Version", "2.5.13") + return confTree + } + // check that the latest version here is the same as in default.go largestKey := getNextVersion(migrations) if largestKey != tomlConfigVersion { diff --git a/cmd/harmony/default.go b/cmd/harmony/default.go index 3835c49210..95e05b29cc 100644 --- a/cmd/harmony/default.go +++ b/cmd/harmony/default.go @@ -5,7 +5,7 @@ import ( nodeconfig "github.com/harmony-one/harmony/internal/configs/node" ) -const tomlConfigVersion = "2.5.12" +const tomlConfigVersion = "2.5.13" const ( defNetworkType = nodeconfig.Mainnet @@ -44,6 +44,9 @@ var defaultConfig = harmonyconfig.HarmonyConfig{ Port: nodeconfig.DefaultRPCPort, AuthPort: nodeconfig.DefaultAuthRPCPort, RosettaPort: nodeconfig.DefaultRosettaPort, + ReadTimeout: nodeconfig.DefaultHTTPTimeoutRead, + WriteTimeout: nodeconfig.DefaultHTTPTimeoutWrite, + IdleTimeout: nodeconfig.DefaultHTTPTimeoutIdle, }, WS: harmonyconfig.WsConfig{ Enabled: true, @@ -59,6 +62,7 @@ var defaultConfig = harmonyconfig.HarmonyConfig{ RpcFilterFile: "./.hmy/rpc_filter.txt", RateLimterEnabled: true, RequestsPerSecond: nodeconfig.DefaultRPCRateLimit, + EvmCallTimeout: nodeconfig.DefaultEvmCallTimeout, }, BLSKeys: harmonyconfig.BlsConfig{ KeyDir: "./.hmy/blskeys", diff --git a/cmd/harmony/flags.go b/cmd/harmony/flags.go index 2ffceb6e5b..8a2799ce28 100644 --- a/cmd/harmony/flags.go +++ b/cmd/harmony/flags.go @@ -75,6 +75,9 @@ var ( httpPortFlag, httpAuthPortFlag, httpRosettaPortFlag, + httpReadTimeoutFlag, + httpWriteTimeoutFlag, + httpIdleTimeoutFlag, } wsFlags = []cli.Flag{ @@ -92,6 +95,7 @@ var ( rpcFilterFileFlag, rpcRateLimiterEnabledFlag, rpcRateLimitFlag, + rpcEvmCallTimeoutFlag, } blsFlags = append(newBLSFlags, legacyBLSFlags...) @@ -695,6 +699,21 @@ var ( Usage: "rosetta port to listen for HTTP requests", DefValue: defaultConfig.HTTP.RosettaPort, } + httpReadTimeoutFlag = cli.StringFlag{ + Name: "http.timeout.read", + Usage: "maximum duration to read the entire request, including the body", + DefValue: defaultConfig.HTTP.ReadTimeout, + } + httpWriteTimeoutFlag = cli.StringFlag{ + Name: "http.timeout.write", + Usage: "maximum duration before timing out writes of the response", + DefValue: defaultConfig.HTTP.WriteTimeout, + } + httpIdleTimeoutFlag = cli.StringFlag{ + Name: "http.timeout.idle", + Usage: "maximum amount of time to wait for the next request when keep-alives are enabled", + DefValue: defaultConfig.HTTP.IdleTimeout, + } ) func applyHTTPFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { @@ -732,6 +751,16 @@ func applyHTTPFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { config.HTTP.Enabled = true } + if cli.IsFlagChanged(cmd, httpReadTimeoutFlag) { + config.HTTP.ReadTimeout = cli.GetStringFlagValue(cmd, httpReadTimeoutFlag) + } + if cli.IsFlagChanged(cmd, httpWriteTimeoutFlag) { + config.HTTP.WriteTimeout = cli.GetStringFlagValue(cmd, httpWriteTimeoutFlag) + } + if cli.IsFlagChanged(cmd, httpIdleTimeoutFlag) { + config.HTTP.IdleTimeout = cli.GetStringFlagValue(cmd, httpIdleTimeoutFlag) + } + } // ws flags @@ -821,6 +850,12 @@ var ( Usage: "the number of requests per second for RPCs", DefValue: defaultConfig.RPCOpt.RequestsPerSecond, } + + rpcEvmCallTimeoutFlag = cli.StringFlag{ + Name: "rpc.evm-call-timeout", + Usage: "timeout for evm execution (eth_call); 0 means infinite timeout", + DefValue: defaultConfig.RPCOpt.EvmCallTimeout, + } ) func applyRPCOptFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { @@ -845,7 +880,9 @@ func applyRPCOptFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { if cli.IsFlagChanged(cmd, rpcRateLimitFlag) { config.RPCOpt.RequestsPerSecond = cli.GetIntFlagValue(cmd, rpcRateLimitFlag) } - + if cli.IsFlagChanged(cmd, rpcEvmCallTimeoutFlag) { + config.RPCOpt.EvmCallTimeout = cli.GetStringFlagValue(cmd, rpcEvmCallTimeoutFlag) + } } // bls flags diff --git a/cmd/harmony/flags_test.go b/cmd/harmony/flags_test.go index 15ba9aaf08..2015188ed4 100644 --- a/cmd/harmony/flags_test.go +++ b/cmd/harmony/flags_test.go @@ -77,6 +77,9 @@ func TestHarmonyFlags(t *testing.T) { AuthPort: 9501, RosettaEnabled: false, RosettaPort: 9700, + ReadTimeout: defaultConfig.HTTP.ReadTimeout, + WriteTimeout: defaultConfig.HTTP.WriteTimeout, + IdleTimeout: defaultConfig.HTTP.IdleTimeout, }, RPCOpt: harmonyconfig.RpcOptConfig{ DebugEnabled: false, @@ -86,6 +89,7 @@ func TestHarmonyFlags(t *testing.T) { RpcFilterFile: "./.hmy/rpc_filter.txt", RateLimterEnabled: true, RequestsPerSecond: 1000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, }, WS: harmonyconfig.WsConfig{ Enabled: true, @@ -531,6 +535,9 @@ func TestRPCFlags(t *testing.T) { Port: defaultConfig.HTTP.Port, AuthPort: defaultConfig.HTTP.AuthPort, RosettaPort: defaultConfig.HTTP.RosettaPort, + ReadTimeout: defaultConfig.HTTP.ReadTimeout, + WriteTimeout: defaultConfig.HTTP.WriteTimeout, + IdleTimeout: defaultConfig.HTTP.IdleTimeout, }, }, { @@ -542,6 +549,9 @@ func TestRPCFlags(t *testing.T) { Port: 9001, AuthPort: defaultConfig.HTTP.AuthPort, RosettaPort: defaultConfig.HTTP.RosettaPort, + ReadTimeout: defaultConfig.HTTP.ReadTimeout, + WriteTimeout: defaultConfig.HTTP.WriteTimeout, + IdleTimeout: defaultConfig.HTTP.IdleTimeout, }, }, { @@ -553,6 +563,9 @@ func TestRPCFlags(t *testing.T) { Port: defaultConfig.HTTP.Port, AuthPort: 9001, RosettaPort: defaultConfig.HTTP.RosettaPort, + ReadTimeout: defaultConfig.HTTP.ReadTimeout, + WriteTimeout: defaultConfig.HTTP.WriteTimeout, + IdleTimeout: defaultConfig.HTTP.IdleTimeout, }, }, { @@ -564,6 +577,9 @@ func TestRPCFlags(t *testing.T) { Port: 9001, AuthPort: defaultConfig.HTTP.AuthPort, RosettaPort: 10001, + ReadTimeout: defaultConfig.HTTP.ReadTimeout, + WriteTimeout: defaultConfig.HTTP.WriteTimeout, + IdleTimeout: defaultConfig.HTTP.IdleTimeout, }, }, { @@ -575,6 +591,9 @@ func TestRPCFlags(t *testing.T) { Port: defaultConfig.HTTP.Port, AuthPort: defaultConfig.HTTP.AuthPort, RosettaPort: 10001, + ReadTimeout: defaultConfig.HTTP.ReadTimeout, + WriteTimeout: defaultConfig.HTTP.WriteTimeout, + IdleTimeout: defaultConfig.HTTP.IdleTimeout, }, }, { @@ -586,6 +605,23 @@ func TestRPCFlags(t *testing.T) { Port: 9501, AuthPort: 9502, RosettaPort: 9701, + ReadTimeout: defaultConfig.HTTP.ReadTimeout, + WriteTimeout: defaultConfig.HTTP.WriteTimeout, + IdleTimeout: defaultConfig.HTTP.IdleTimeout, + }, + }, + { + args: []string{"--http.timeout.read", "10s", "--http.timeout.write", "20s", "--http.timeout.idle", "30s"}, + expConfig: harmonyconfig.HttpConfig{ + Enabled: true, + RosettaEnabled: false, + IP: defaultConfig.HTTP.IP, + Port: defaultConfig.HTTP.Port, + AuthPort: defaultConfig.HTTP.AuthPort, + RosettaPort: defaultConfig.HTTP.RosettaPort, + ReadTimeout: "10s", + WriteTimeout: "20s", + IdleTimeout: "30s", }, }, } @@ -699,6 +735,7 @@ func TestRPCOptFlags(t *testing.T) { RpcFilterFile: "./.hmy/rpc_filter.txt", RateLimterEnabled: true, RequestsPerSecond: 1000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, }, }, @@ -712,6 +749,7 @@ func TestRPCOptFlags(t *testing.T) { RpcFilterFile: "./.hmy/rpc_filter.txt", RateLimterEnabled: true, RequestsPerSecond: 1000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, }, }, @@ -725,6 +763,7 @@ func TestRPCOptFlags(t *testing.T) { RpcFilterFile: "./.hmy/rpc_filter.txt", RateLimterEnabled: true, RequestsPerSecond: 1000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, }, }, @@ -738,6 +777,7 @@ func TestRPCOptFlags(t *testing.T) { RpcFilterFile: "./.hmy/rpc_filter.txt", RateLimterEnabled: true, RequestsPerSecond: 1000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, }, }, @@ -751,6 +791,7 @@ func TestRPCOptFlags(t *testing.T) { RpcFilterFile: "./rmf.toml", RateLimterEnabled: true, RequestsPerSecond: 1000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, }, }, @@ -764,6 +805,7 @@ func TestRPCOptFlags(t *testing.T) { RpcFilterFile: "./.hmy/rpc_filter.txt", RateLimterEnabled: true, RequestsPerSecond: 1000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, }, }, @@ -777,6 +819,7 @@ func TestRPCOptFlags(t *testing.T) { RpcFilterFile: "./.hmy/rpc_filter.txt", RateLimterEnabled: true, RequestsPerSecond: 2000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, }, }, @@ -790,6 +833,21 @@ func TestRPCOptFlags(t *testing.T) { RpcFilterFile: "./.hmy/rpc_filter.txt", RateLimterEnabled: false, RequestsPerSecond: 2000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, + }, + }, + + { + args: []string{"--rpc.evm-call-timeout", "10s"}, + expConfig: harmonyconfig.RpcOptConfig{ + DebugEnabled: false, + EthRPCsEnabled: true, + StakingRPCsEnabled: true, + LegacyRPCsEnabled: true, + RpcFilterFile: "./.hmy/rpc_filter.txt", + RateLimterEnabled: true, + RequestsPerSecond: 1000, + EvmCallTimeout: "10s", }, }, } diff --git a/internal/configs/harmony/harmony.go b/internal/configs/harmony/harmony.go index 67c29f820f..d4e8df4b00 100644 --- a/internal/configs/harmony/harmony.go +++ b/internal/configs/harmony/harmony.go @@ -3,6 +3,10 @@ package harmony import ( "reflect" "strings" + "time" + + nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + "github.com/harmony-one/harmony/internal/utils" ) // HarmonyConfig contains all the configs user can set for running harmony binary. Served as the bridge @@ -32,6 +36,62 @@ type HarmonyConfig struct { ShardData ShardDataConfig } +func (hc HarmonyConfig) ToRPCServerConfig() nodeconfig.RPCServerConfig { + readTimeout, err := time.ParseDuration(hc.HTTP.ReadTimeout) + if err != nil { + readTimeout, _ = time.ParseDuration(nodeconfig.DefaultHTTPTimeoutRead) + utils.Logger().Warn(). + Str("provided", hc.HTTP.ReadTimeout). + Dur("updated", readTimeout). + Msg("Sanitizing invalid http read timeout") + } + writeTimeout, err := time.ParseDuration(hc.HTTP.WriteTimeout) + if err != nil { + writeTimeout, _ = time.ParseDuration(nodeconfig.DefaultHTTPTimeoutWrite) + utils.Logger().Warn(). + Str("provided", hc.HTTP.WriteTimeout). + Dur("updated", writeTimeout). + Msg("Sanitizing invalid http write timeout") + } + idleTimeout, err := time.ParseDuration(hc.HTTP.IdleTimeout) + if err != nil { + idleTimeout, _ = time.ParseDuration(nodeconfig.DefaultHTTPTimeoutIdle) + utils.Logger().Warn(). + Str("provided", hc.HTTP.IdleTimeout). + Dur("updated", idleTimeout). + Msg("Sanitizing invalid http idle timeout") + } + evmCallTimeout, err := time.ParseDuration(hc.RPCOpt.EvmCallTimeout) + if err != nil { + evmCallTimeout, _ = time.ParseDuration(nodeconfig.DefaultEvmCallTimeout) + utils.Logger().Warn(). + Str("provided", hc.RPCOpt.EvmCallTimeout). + Dur("updated", evmCallTimeout). + Msg("Sanitizing invalid evm_call timeout") + } + return nodeconfig.RPCServerConfig{ + HTTPEnabled: hc.HTTP.Enabled, + HTTPIp: hc.HTTP.IP, + HTTPPort: hc.HTTP.Port, + HTTPAuthPort: hc.HTTP.AuthPort, + HTTPTimeoutRead: readTimeout, + HTTPTimeoutWrite: writeTimeout, + HTTPTimeoutIdle: idleTimeout, + WSEnabled: hc.WS.Enabled, + WSIp: hc.WS.IP, + WSPort: hc.WS.Port, + WSAuthPort: hc.WS.AuthPort, + DebugEnabled: hc.RPCOpt.DebugEnabled, + EthRPCsEnabled: hc.RPCOpt.EthRPCsEnabled, + StakingRPCsEnabled: hc.RPCOpt.StakingRPCsEnabled, + LegacyRPCsEnabled: hc.RPCOpt.LegacyRPCsEnabled, + RpcFilterFile: hc.RPCOpt.RpcFilterFile, + RateLimiterEnabled: hc.RPCOpt.RateLimterEnabled, + RequestsPerSecond: hc.RPCOpt.RequestsPerSecond, + EvmCallTimeout: evmCallTimeout, + } +} + type DnsSync struct { Port int // replaces: Network.DNSSyncPort Zone string // replaces: Network.DNSZone @@ -180,6 +240,9 @@ type HttpConfig struct { AuthPort int RosettaEnabled bool RosettaPort int + ReadTimeout string + WriteTimeout string + IdleTimeout string } type WsConfig struct { @@ -197,6 +260,7 @@ type RpcOptConfig struct { RpcFilterFile string // Define filters to enable/disable RPC exposure RateLimterEnabled bool // Enable Rate limiter for RPC RequestsPerSecond int // for RPC rate limiter + EvmCallTimeout string // Timeout for eth_call } type DevnetConfig struct { diff --git a/internal/configs/harmony/harmony_test.go b/internal/configs/harmony/harmony_test.go new file mode 100644 index 0000000000..fef7cac9df --- /dev/null +++ b/internal/configs/harmony/harmony_test.go @@ -0,0 +1,81 @@ +package harmony + +import ( + "fmt" + "testing" + "time" + + nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + "github.com/stretchr/testify/assert" +) + +func TestToRPCServerConfig(t *testing.T) { + tests := []struct { + input HarmonyConfig + output nodeconfig.RPCServerConfig + }{ + { + input: HarmonyConfig{ + HTTP: HttpConfig{ + Enabled: true, + RosettaEnabled: false, + IP: "127.0.0.1", + Port: nodeconfig.DefaultRPCPort, + AuthPort: nodeconfig.DefaultAuthRPCPort, + RosettaPort: nodeconfig.DefaultRosettaPort, + ReadTimeout: "-1", + WriteTimeout: "-2", + IdleTimeout: "-3", + }, + WS: WsConfig{ + Enabled: true, + IP: "127.0.0.1", + Port: nodeconfig.DefaultWSPort, + AuthPort: nodeconfig.DefaultAuthWSPort, + }, + RPCOpt: RpcOptConfig{ + DebugEnabled: false, + EthRPCsEnabled: true, + StakingRPCsEnabled: true, + LegacyRPCsEnabled: true, + RpcFilterFile: "./.hmy/rpc_filter.txt", + RateLimterEnabled: true, + RequestsPerSecond: nodeconfig.DefaultRPCRateLimit, + EvmCallTimeout: "-4", + }, + }, + output: nodeconfig.RPCServerConfig{ + HTTPEnabled: true, + HTTPIp: "127.0.0.1", + HTTPPort: nodeconfig.DefaultRPCPort, + HTTPAuthPort: nodeconfig.DefaultAuthRPCPort, + HTTPTimeoutRead: 30 * time.Second, + HTTPTimeoutWrite: 30 * time.Second, + HTTPTimeoutIdle: 120 * time.Second, + WSEnabled: true, + WSIp: "127.0.0.1", + WSPort: nodeconfig.DefaultWSPort, + WSAuthPort: nodeconfig.DefaultAuthWSPort, + DebugEnabled: false, + EthRPCsEnabled: true, + StakingRPCsEnabled: true, + LegacyRPCsEnabled: true, + RpcFilterFile: "./.hmy/rpc_filter.txt", + RateLimiterEnabled: true, + RequestsPerSecond: nodeconfig.DefaultRPCRateLimit, + EvmCallTimeout: 5 * time.Second, + }, + }, + } + for i, tt := range tests { + assertObject := assert.New(t) + name := fmt.Sprintf("TestToRPCServerConfig: #%d", i) + t.Run(name, func(t *testing.T) { + assertObject.Equal( + tt.input.ToRPCServerConfig(), + tt.output, + name, + ) + }) + } +} diff --git a/internal/configs/node/config.go b/internal/configs/node/config.go index 9a0e950ecc..9f681fca9f 100644 --- a/internal/configs/node/config.go +++ b/internal/configs/node/config.go @@ -8,6 +8,7 @@ import ( "math/big" "strings" "sync" + "time" bls_core "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/crypto/bls" @@ -115,6 +116,10 @@ type RPCServerConfig struct { HTTPPort int HTTPAuthPort int + HTTPTimeoutRead time.Duration + HTTPTimeoutWrite time.Duration + HTTPTimeoutIdle time.Duration + WSEnabled bool WSIp string WSPort int @@ -130,6 +135,8 @@ type RPCServerConfig struct { RateLimiterEnabled bool RequestsPerSecond int + + EvmCallTimeout time.Duration } // RosettaServerConfig is the config for the rosetta server diff --git a/internal/configs/node/network.go b/internal/configs/node/network.go index 8b15d3359c..03f1472125 100644 --- a/internal/configs/node/network.go +++ b/internal/configs/node/network.go @@ -51,6 +51,12 @@ const ( DefaultAuthRPCPort = 9501 // DefaultRosettaPort is the default rosetta port. The actual port used is 9000+700 DefaultRosettaPort = 9700 + // DefaultHTTP timeouts - read, write, and idle + DefaultHTTPTimeoutRead = "30s" + DefaultHTTPTimeoutWrite = "30s" + DefaultHTTPTimeoutIdle = "120s" + // DefaultEvmCallTimeout is the default timeout for evm call + DefaultEvmCallTimeout = "5s" // DefaultWSPort is the default port for web socket endpoint. The actual port used is DefaultWSPort = 9800 // DefaultAuthWSPort is the default port for web socket auth endpoint. The actual port used is diff --git a/rosetta/rosetta.go b/rosetta/rosetta.go index 860c2d0d66..5ff222a65b 100644 --- a/rosetta/rosetta.go +++ b/rosetta/rosetta.go @@ -84,7 +84,10 @@ func getRouter(asserter *asserter.Asserter, hmy *hmy.Harmony, limiterEnable bool server.NewMempoolAPIController(services.NewMempoolAPI(hmy), asserter), server.NewNetworkAPIController(services.NewNetworkAPI(hmy), asserter), server.NewConstructionAPIController(services.NewConstructionAPI(hmy), asserter), - server.NewCallAPIController(services.NewCallAPIService(hmy, limiterEnable, rateLimit), asserter), + server.NewCallAPIController( + services.NewCallAPIService(hmy, limiterEnable, rateLimit, hmy.NodeAPI.GetConfig().NodeConfig.RPCServer.EvmCallTimeout), + asserter, + ), server.NewEventsAPIController(services.NewEventAPI(hmy), asserter), server.NewSearchAPIController(services.NewSearchAPI(hmy), asserter), ) diff --git a/rosetta/services/call_service.go b/rosetta/services/call_service.go index 46f528d7b9..9d26bab282 100644 --- a/rosetta/services/call_service.go +++ b/rosetta/services/call_service.go @@ -3,6 +3,7 @@ package services import ( "context" "encoding/json" + "time" "github.com/coinbase/rosetta-sdk-go/server" "github.com/coinbase/rosetta-sdk-go/types" @@ -82,10 +83,15 @@ func (c *CallAPIService) Call( } -func NewCallAPIService(hmy *hmy.Harmony, limiterEnable bool, rateLimit int) server.CallAPIServicer { +func NewCallAPIService( + hmy *hmy.Harmony, + limiterEnable bool, + rateLimit int, + evmCallTimeout time.Duration, +) server.CallAPIServicer { return &CallAPIService{ hmy: hmy, - publicContractAPI: rpc2.NewPublicContractAPI(hmy, rpc2.V2, limiterEnable, rateLimit), + publicContractAPI: rpc2.NewPublicContractAPI(hmy, rpc2.V2, limiterEnable, rateLimit, evmCallTimeout), publicStakingAPI: rpc2.NewPublicStakingAPI(hmy, rpc2.V2), publicBlockChainAPI: rpc2.NewPublicBlockchainAPI(hmy, rpc2.V2, limiterEnable, rateLimit), } diff --git a/rpc/contract.go b/rpc/contract.go index 337bea7cde..abcb4f9418 100644 --- a/rpc/contract.go +++ b/rpc/contract.go @@ -31,11 +31,18 @@ type PublicContractService struct { hmy *hmy.Harmony version Version // TEMP SOLUTION to rpc node spamming issue - limiterCall *rate.Limiter + limiterCall *rate.Limiter + evmCallTimeout time.Duration } // NewPublicContractAPI creates a new API for the RPC interface -func NewPublicContractAPI(hmy *hmy.Harmony, version Version, limiterEnable bool, limit int) rpc.API { +func NewPublicContractAPI( + hmy *hmy.Harmony, + version Version, + limiterEnable bool, + limit int, + evmCallTimeout time.Duration, +) rpc.API { var limiter *rate.Limiter if limiterEnable { limiter = rate.NewLimiter(rate.Limit(limit), limit) @@ -44,8 +51,13 @@ func NewPublicContractAPI(hmy *hmy.Harmony, version Version, limiterEnable bool, return rpc.API{ Namespace: version.Namespace(), Version: APIVersion, - Service: &PublicContractService{hmy, version, limiter}, - Public: true, + Service: &PublicContractService{ + hmy: hmy, + version: version, + limiterCall: limiter, + evmCallTimeout: evmCallTimeout, + }, + Public: true, } } @@ -80,7 +92,7 @@ func (s *PublicContractService) Call( } // Execute call - result, err := DoEVMCall(ctx, s.hmy, args, blockNrOrHash, CallTimeout) + result, err := DoEVMCall(ctx, s.hmy, args, blockNrOrHash, s.evmCallTimeout) if err != nil { return nil, err } diff --git a/rpc/rpc.go b/rpc/rpc.go index 2c94f0a5e5..1bae7367de 100644 --- a/rpc/rpc.go +++ b/rpc/rpc.go @@ -58,9 +58,9 @@ var ( wsEndpoint = "" wsAuthEndpoint = "" httpVirtualHosts = []string{"*"} - httpTimeouts = rpc.DefaultHTTPTimeouts - httpOrigins = []string{"*"} - wsOrigins = []string{"*"} + // httpTimeouts = rpc.DefaultHTTPTimeouts + httpOrigins = []string{"*"} + wsOrigins = []string{"*"} ) // Version of the RPC @@ -86,13 +86,18 @@ func StartServers(hmy *hmy.Harmony, apis []rpc.API, config nodeconfig.RPCServerC rmf.ExposeAll() } if config.HTTPEnabled { + timeouts := rpc.HTTPTimeouts{ + ReadTimeout: config.HTTPTimeoutRead, + WriteTimeout: config.HTTPTimeoutWrite, + IdleTimeout: config.HTTPTimeoutIdle, + } httpEndpoint = fmt.Sprintf("%v:%v", config.HTTPIp, config.HTTPPort) - if err := startHTTP(apis, &rmf); err != nil { + if err := startHTTP(apis, &rmf, timeouts); err != nil { return err } httpAuthEndpoint = fmt.Sprintf("%v:%v", config.HTTPIp, config.HTTPAuthPort) - if err := startAuthHTTP(authApis, &rmf); err != nil { + if err := startAuthHTTP(authApis, &rmf, timeouts); err != nil { return err } } @@ -158,8 +163,8 @@ func getAPIs(hmy *hmy.Harmony, config nodeconfig.RPCServerConfig) []rpc.API { NewPublicHarmonyAPI(hmy, V2), NewPublicBlockchainAPI(hmy, V1, config.RateLimiterEnabled, config.RequestsPerSecond), NewPublicBlockchainAPI(hmy, V2, config.RateLimiterEnabled, config.RequestsPerSecond), - NewPublicContractAPI(hmy, V1, config.RateLimiterEnabled, config.RequestsPerSecond), - NewPublicContractAPI(hmy, V2, config.RateLimiterEnabled, config.RequestsPerSecond), + NewPublicContractAPI(hmy, V1, config.RateLimiterEnabled, config.RequestsPerSecond, config.EvmCallTimeout), + NewPublicContractAPI(hmy, V2, config.RateLimiterEnabled, config.RequestsPerSecond, config.EvmCallTimeout), NewPublicTransactionAPI(hmy, V1), NewPublicTransactionAPI(hmy, V2), NewPublicPoolAPI(hmy, V1, config.RateLimiterEnabled, config.RequestsPerSecond), @@ -185,7 +190,7 @@ func getAPIs(hmy *hmy.Harmony, config nodeconfig.RPCServerConfig) []rpc.API { publicAPIs = append(publicAPIs, NewPublicHarmonyAPI(hmy, Eth), NewPublicBlockchainAPI(hmy, Eth, config.RateLimiterEnabled, config.RequestsPerSecond), - NewPublicContractAPI(hmy, Eth, config.RateLimiterEnabled, config.RequestsPerSecond), + NewPublicContractAPI(hmy, Eth, config.RateLimiterEnabled, config.RequestsPerSecond, config.EvmCallTimeout), NewPublicTransactionAPI(hmy, Eth), NewPublicPoolAPI(hmy, Eth, config.RateLimiterEnabled, config.RequestsPerSecond), eth.NewPublicEthService(hmy, "eth"), @@ -210,7 +215,7 @@ func getAPIs(hmy *hmy.Harmony, config nodeconfig.RPCServerConfig) []rpc.API { return publicAPIs } -func startHTTP(apis []rpc.API, rmf *rpc.RpcMethodFilter) (err error) { +func startHTTP(apis []rpc.API, rmf *rpc.RpcMethodFilter, httpTimeouts rpc.HTTPTimeouts) (err error) { httpListener, httpHandler, err = rpc.StartHTTPEndpoint( httpEndpoint, apis, HTTPModules, rmf, httpOrigins, httpVirtualHosts, httpTimeouts, ) @@ -227,7 +232,7 @@ func startHTTP(apis []rpc.API, rmf *rpc.RpcMethodFilter) (err error) { return nil } -func startAuthHTTP(apis []rpc.API, rmf *rpc.RpcMethodFilter) (err error) { +func startAuthHTTP(apis []rpc.API, rmf *rpc.RpcMethodFilter, httpTimeouts rpc.HTTPTimeouts) (err error) { httpListener, httpHandler, err = rpc.StartHTTPEndpoint( httpAuthEndpoint, apis, HTTPModules, rmf, httpOrigins, httpVirtualHosts, httpTimeouts, ) From 2bd16b8df6fb63ccada09c363b43ecbb7a91633a Mon Sep 17 00:00:00 2001 From: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Date: Thu, 23 Feb 2023 09:24:23 +0000 Subject: [PATCH 278/420] remove default timeouts --- rosetta/infra/harmony-mainnet.conf | 26 +++++++++++++++++++++----- rosetta/infra/harmony-pstn.conf | 26 +++++++++++++++++++++----- rosetta/services/construction_check.go | 2 +- rpc/rpc.go | 8 ++------ 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/rosetta/infra/harmony-mainnet.conf b/rosetta/infra/harmony-mainnet.conf index a929eeeab1..8d51609cb1 100644 --- a/rosetta/infra/harmony-mainnet.conf +++ b/rosetta/infra/harmony-mainnet.conf @@ -1,4 +1,4 @@ -Version = "2.5.11" +Version = "2.5.13" [BLSKeys] KMSConfigFile = "" @@ -41,9 +41,12 @@ Version = "2.5.11" AuthPort = 9501 Enabled = true IP = "0.0.0.0" + IdleTimeout = "120s" Port = 9500 + ReadTimeout = "30s" RosettaEnabled = true RosettaPort = 9700 + WriteTimeout = "30s" [Log] Console = false @@ -62,6 +65,8 @@ Version = "2.5.11" NetworkType = "mainnet" [P2P] + ConnManagerHighWatermark = 192 + ConnManagerLowWatermark = 160 DisablePrivateIPScan = false DiscConcurrency = 0 IP = "0.0.0.0" @@ -69,8 +74,6 @@ Version = "2.5.11" MaxConnsPerIP = 10 MaxPeers = 0 Port = 9000 - ConnManagerLowWatermark = 160 - ConnManagerHighWatermark = 192 WaitForEachPeerToConnect = false [Pprof] @@ -84,6 +87,7 @@ Version = "2.5.11" [RPCOpt] DebugEnabled = false EthRPCsEnabled = true + EvmCallTimeout = "5s" LegacyRPCsEnabled = true RateLimterEnabled = true RequestsPerSecond = 1000 @@ -104,11 +108,23 @@ Version = "2.5.11" DiscHighCap = 128 DiscSoftLowCap = 8 Downloader = false - StagedSync = false Enabled = false InitStreams = 8 - MinPeers = 5 MaxAdvertiseWaitTime = 30 + MinPeers = 5 + StagedSync = false + + [Sync.StagedSyncCfg] + DoubleCheckBlockHashes = false + InsertChainBatchSize = 0 + LogProgress = false + MaxBackgroundBlocks = 0 + MaxBlocksPerSyncCycle = 0 + MaxMemSyncCycleSize = 0 + TurboMode = false + UseMemDB = false + VerifyAllSig = false + VerifyHeaderBatchSize = 0 [TxPool] AccountSlots = 16 diff --git a/rosetta/infra/harmony-pstn.conf b/rosetta/infra/harmony-pstn.conf index edb911f87a..1bb865c1a5 100644 --- a/rosetta/infra/harmony-pstn.conf +++ b/rosetta/infra/harmony-pstn.conf @@ -1,4 +1,4 @@ -Version = "2.5.11" +Version = "2.5.13" [BLSKeys] KMSConfigFile = "" @@ -41,9 +41,12 @@ Version = "2.5.11" AuthPort = 9501 Enabled = true IP = "0.0.0.0" + IdleTimeout = "120s" Port = 9500 + ReadTimeout = "30s" RosettaEnabled = true RosettaPort = 9700 + WriteTimeout = "30s" [Log] Console = false @@ -62,6 +65,8 @@ Version = "2.5.11" NetworkType = "partner" [P2P] + ConnManagerHighWatermark = 192 + ConnManagerLowWatermark = 160 DisablePrivateIPScan = false DiscConcurrency = 0 IP = "0.0.0.0" @@ -69,8 +74,6 @@ Version = "2.5.11" MaxConnsPerIP = 10 MaxPeers = 0 Port = 9000 - ConnManagerLowWatermark = 160 - ConnManagerHighWatermark = 192 WaitForEachPeerToConnect = false [Pprof] @@ -84,6 +87,7 @@ Version = "2.5.11" [RPCOpt] DebugEnabled = false EthRPCsEnabled = true + EvmCallTimeout = "5s" LegacyRPCsEnabled = true RateLimterEnabled = true RequestsPerSecond = 1000 @@ -104,11 +108,23 @@ Version = "2.5.11" DiscHighCap = 128 DiscSoftLowCap = 8 Downloader = false - StagedSync = false Enabled = false InitStreams = 8 - MinPeers = 2 MaxAdvertiseWaitTime = 30 + MinPeers = 2 + StagedSync = false + + [Sync.StagedSyncCfg] + DoubleCheckBlockHashes = false + InsertChainBatchSize = 0 + LogProgress = false + MaxBackgroundBlocks = 0 + MaxBlocksPerSyncCycle = 0 + MaxMemSyncCycleSize = 0 + TurboMode = false + UseMemDB = false + VerifyAllSig = false + VerifyHeaderBatchSize = 0 [TxPool] AccountSlots = 16 diff --git a/rosetta/services/construction_check.go b/rosetta/services/construction_check.go index 1c3e1db699..f60296971c 100644 --- a/rosetta/services/construction_check.go +++ b/rosetta/services/construction_check.go @@ -270,7 +270,7 @@ func (s *ConstructAPI) ConstructionMetadata( callArgs.To = &contractAddress } evmExe, err := rpc.DoEVMCall( - ctx, s.hmy, callArgs, latest, rpc.CallTimeout, + ctx, s.hmy, callArgs, latest, s.hmy.NodeAPI.GetConfig().NodeConfig.RPCServer.EvmCallTimeout, ) if err != nil { return nil, common.NewError(common.CatchAllError, map[string]interface{}{ diff --git a/rpc/rpc.go b/rpc/rpc.go index 1bae7367de..a8f1e121a9 100644 --- a/rpc/rpc.go +++ b/rpc/rpc.go @@ -4,7 +4,6 @@ import ( "fmt" "net" "strings" - "time" "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy" @@ -28,8 +27,6 @@ const ( const ( // APIVersion used for DApp's, bumped after RPC refactor (7/2020) APIVersion = "1.1" - // CallTimeout is the timeout given to all contract calls - CallTimeout = 5 * time.Second // LogTag is the tag found in the log for all RPC logs LogTag = "[RPC]" // HTTPPortOffset .. @@ -58,9 +55,8 @@ var ( wsEndpoint = "" wsAuthEndpoint = "" httpVirtualHosts = []string{"*"} - // httpTimeouts = rpc.DefaultHTTPTimeouts - httpOrigins = []string{"*"} - wsOrigins = []string{"*"} + httpOrigins = []string{"*"} + wsOrigins = []string{"*"} ) // Version of the RPC From 77563d6974ecf38ac9fb5cefbd03f59ab02a87e2 Mon Sep 17 00:00:00 2001 From: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Date: Thu, 23 Feb 2023 09:29:06 +0000 Subject: [PATCH 279/420] store the evm call timeout in rosetta object --- rosetta/rosetta.go | 3 ++- rosetta/services/construction.go | 15 +++++++++------ rosetta/services/construction_check.go | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/rosetta/rosetta.go b/rosetta/rosetta.go index 5ff222a65b..a056300573 100644 --- a/rosetta/rosetta.go +++ b/rosetta/rosetta.go @@ -85,7 +85,8 @@ func getRouter(asserter *asserter.Asserter, hmy *hmy.Harmony, limiterEnable bool server.NewNetworkAPIController(services.NewNetworkAPI(hmy), asserter), server.NewConstructionAPIController(services.NewConstructionAPI(hmy), asserter), server.NewCallAPIController( - services.NewCallAPIService(hmy, limiterEnable, rateLimit, hmy.NodeAPI.GetConfig().NodeConfig.RPCServer.EvmCallTimeout), + services.NewCallAPIService(hmy, limiterEnable, rateLimit, + hmy.NodeAPI.GetConfig().NodeConfig.RPCServer.EvmCallTimeout), asserter, ), server.NewEventsAPIController(services.NewEventAPI(hmy), asserter), diff --git a/rosetta/services/construction.go b/rosetta/services/construction.go index 3442d4c629..f2bd6e6c91 100644 --- a/rosetta/services/construction.go +++ b/rosetta/services/construction.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/big" + "time" "github.com/coinbase/rosetta-sdk-go/server" "github.com/coinbase/rosetta-sdk-go/types" @@ -24,17 +25,19 @@ const ( // ConstructAPI implements the server.ConstructAPIServicer interface. type ConstructAPI struct { - hmy *hmy.Harmony - signer hmyTypes.Signer - stakingSigner stakingTypes.Signer + hmy *hmy.Harmony + signer hmyTypes.Signer + stakingSigner stakingTypes.Signer + evmCallTimeout time.Duration } // NewConstructionAPI creates a new instance of a ConstructAPI. func NewConstructionAPI(hmy *hmy.Harmony) server.ConstructionAPIServicer { return &ConstructAPI{ - hmy: hmy, - signer: hmyTypes.NewEIP155Signer(new(big.Int).SetUint64(hmy.ChainID)), - stakingSigner: stakingTypes.NewEIP155Signer(new(big.Int).SetUint64(hmy.ChainID)), + hmy: hmy, + signer: hmyTypes.NewEIP155Signer(new(big.Int).SetUint64(hmy.ChainID)), + stakingSigner: stakingTypes.NewEIP155Signer(new(big.Int).SetUint64(hmy.ChainID)), + evmCallTimeout: hmy.NodeAPI.GetConfig().NodeConfig.RPCServer.EvmCallTimeout, } } diff --git a/rosetta/services/construction_check.go b/rosetta/services/construction_check.go index f60296971c..c842770ab7 100644 --- a/rosetta/services/construction_check.go +++ b/rosetta/services/construction_check.go @@ -270,7 +270,7 @@ func (s *ConstructAPI) ConstructionMetadata( callArgs.To = &contractAddress } evmExe, err := rpc.DoEVMCall( - ctx, s.hmy, callArgs, latest, s.hmy.NodeAPI.GetConfig().NodeConfig.RPCServer.EvmCallTimeout, + ctx, s.hmy, callArgs, latest, s.evmCallTimeout, ) if err != nil { return nil, common.NewError(common.CatchAllError, map[string]interface{}{ From 1d6259b283f34a18b3c919c290a3ddd14805a63e Mon Sep 17 00:00:00 2001 From: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Date: Mon, 27 Feb 2023 04:11:01 +0000 Subject: [PATCH 280/420] [cmd] actually apply ToRPCServerConfig --- cmd/harmony/main.go | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index eacc392f10..f01cb758ed 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -334,23 +334,7 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { } // Parse RPC config - nodeConfig.RPCServer = nodeconfig.RPCServerConfig{ - HTTPEnabled: hc.HTTP.Enabled, - HTTPIp: hc.HTTP.IP, - HTTPPort: hc.HTTP.Port, - HTTPAuthPort: hc.HTTP.AuthPort, - WSEnabled: hc.WS.Enabled, - WSIp: hc.WS.IP, - WSPort: hc.WS.Port, - WSAuthPort: hc.WS.AuthPort, - DebugEnabled: hc.RPCOpt.DebugEnabled, - EthRPCsEnabled: hc.RPCOpt.EthRPCsEnabled, - StakingRPCsEnabled: hc.RPCOpt.StakingRPCsEnabled, - LegacyRPCsEnabled: hc.RPCOpt.LegacyRPCsEnabled, - RpcFilterFile: hc.RPCOpt.RpcFilterFile, - RateLimiterEnabled: hc.RPCOpt.RateLimterEnabled, - RequestsPerSecond: hc.RPCOpt.RequestsPerSecond, - } + nodeConfig.RPCServer = hc.ToRPCServerConfig() // Parse rosetta config nodeConfig.RosettaServer = nodeconfig.RosettaServerConfig{ From 8d15af1e9192ef8ae81aab360721637fa114d87f Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 21 Feb 2023 15:20:47 -0300 Subject: [PATCH 281/420] Removed unused method. --- consensus/quorum/quorum.go | 1 - 1 file changed, 1 deletion(-) diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 6cf79e549a..da1551cdf9 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -98,7 +98,6 @@ type SignatoryTracker interface { // SignatureReader .. type SignatureReader interface { SignatoryTracker - ReadAllBallots(Phase) []*votepower.Ballot ReadBallot(p Phase, pubkey bls.SerializedPublicKey) *votepower.Ballot TwoThirdsSignersCount() int64 // 96 bytes aggregated signature From 09cf941bba852c43781ce9b39894640ca186967c Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 21 Feb 2023 18:45:17 -0300 Subject: [PATCH 282/420] Rotate external leaders on non-beacon chains. --- consensus/consensus_v2.go | 10 +++++++++- consensus/quorum/quorum.go | 1 + consensus/view_change.go | 16 +++++++++++----- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 6b0d54648d..99bccf7554 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -714,7 +714,15 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { } } // Passed all checks, we can change leader. - wasFound, next := consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) + var ( + wasFound bool + next *bls.PublicKeyWrapper + ) + if consensus.ShardID == shard.BeaconChainShardID { + wasFound, next = consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) + } else { + wasFound, next = consensus.Decider.NthNext(leader, 1) + } if !wasFound { utils.Logger().Error().Msg("Failed to get next leader") return diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index da1551cdf9..aba62fa53f 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -75,6 +75,7 @@ type ParticipantTracker interface { Participants() multibls.PublicKeys IndexOf(bls.SerializedPublicKey) int ParticipantsCount() int64 + NthNext(*bls.PublicKeyWrapper, int) (bool, *bls.PublicKeyWrapper) NthNextHmy(shardingconfig.Instance, *bls.PublicKeyWrapper, int) (bool, *bls.PublicKeyWrapper) NthNextHmyExt(shardingconfig.Instance, *bls.PublicKeyWrapper, int) (bool, *bls.PublicKeyWrapper) FirstParticipant(shardingconfig.Instance) *bls.PublicKeyWrapper diff --git a/consensus/view_change.go b/consensus/view_change.go index d8b1058a04..899a50121d 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -202,11 +202,17 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe // FIXME: rotate leader on harmony nodes only before fully externalization var wasFound bool var next *bls.PublicKeyWrapper - if blockchain != nil && blockchain.Config().IsAllowlistEpoch(epoch) { - wasFound, next = consensus.Decider.NthNextHmyExt( - shard.Schedule.InstanceForEpoch(epoch), - lastLeaderPubKey, - gap) + if consensus.Blockchain().Config().IsLeaderRotation(epoch) { + if consensus.ShardID == shard.BeaconChainShardID { + wasFound, next = consensus.Decider.NthNextHmy( + shard.Schedule.InstanceForEpoch(epoch), + lastLeaderPubKey, + gap) + } else { + wasFound, next = consensus.Decider.NthNext( + lastLeaderPubKey, + gap) + } } else { wasFound, next = consensus.Decider.NthNextHmy( shard.Schedule.InstanceForEpoch(epoch), From e0152304b07c3cc064397e893cb38eb275769dfa Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 21 Feb 2023 22:33:57 -0300 Subject: [PATCH 283/420] Fix nil panic. --- consensus/view_change.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/view_change.go b/consensus/view_change.go index 899a50121d..aafdfd1210 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -202,7 +202,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe // FIXME: rotate leader on harmony nodes only before fully externalization var wasFound bool var next *bls.PublicKeyWrapper - if consensus.Blockchain().Config().IsLeaderRotation(epoch) { + if blockchain != nil && blockchain.Config().IsLeaderRotation(epoch) { if consensus.ShardID == shard.BeaconChainShardID { wasFound, next = consensus.Decider.NthNextHmy( shard.Schedule.InstanceForEpoch(epoch), From d6a3c93aec69b4f433393e905a3cd8700505d34c Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 1 Sep 2022 00:53:16 +0700 Subject: [PATCH 284/420] in progress. --- api/service/explorer/service.go | 48 ++++ cmd/harmony/main.go | 6 +- consensus/checks.go | 3 +- consensus/consensus.go | 2 + consensus/consensus_service.go | 164 +++++------ consensus/consensus_v2.go | 433 ++++++++++++++---------------- consensus/leader.go | 11 + consensus/quorum/quorum.go | 7 +- consensus/validator.go | 49 ++-- consensus/view_change.go | 133 +++++---- hmy/hmy.go | 2 + internal/utils/singleton.go | 1 + node/api.go | 6 + node/node_newblock.go | 4 + test/configs/local-resharding.txt | 31 +-- test/deploy.sh | 5 +- 16 files changed, 482 insertions(+), 423 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index b5ffa1b8b0..d96c1bc56c 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -7,11 +7,13 @@ import ( "fmt" "net" "net/http" + "os" "path" "strconv" "time" "github.com/RoaringBitmap/roaring/roaring64" + "github.com/harmony-one/harmony/internal/common" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" ethCommon "github.com/ethereum/go-ethereum/common" @@ -113,12 +115,17 @@ func (s *Service) Run() *http.Server { s.router = mux.NewRouter() + fmt.Println("++", addr) + // Set up router for addresses. // Fetch addresses request, accepts parameter size: how much addresses to read, // parameter prefix: from which address prefix start s.router.Path("/addresses").Queries("size", "{[0-9]*?}", "prefix", "{[a-zA-Z0-9]*?}").HandlerFunc(s.GetAddresses).Methods("GET") s.router.Path("/addresses").HandlerFunc(s.GetAddresses) s.router.Path("/height").HandlerFunc(s.GetHeight) + s.router.Path("/leader").HandlerFunc(s.GetLeader) + s.router.Path("/blocks").HandlerFunc(s.GetBlocks) + s.router.Path("/halt").HandlerFunc(s.halt) // Set up router for supply info s.router.Path("/burn-addresses").Queries().HandlerFunc(s.GetInaccessibleAddressInfo).Methods("GET") @@ -186,6 +193,47 @@ type HeightResponse struct { S3 uint64 `json:"3,omitempty"` } +func (s *Service) GetLeader(w http.ResponseWriter, r *http.Request) { + if s.backend.IsCurrentlyLeader() { + w.Write([]byte("true ")) + } else { + w.Write([]byte("false")) + } + + keys := "" + for _, p := range s.backend.GetPublicKeys() { + addr := common.Address{} + addrBytes := p.Object.GetAddress() + addr.SetBytes(addrBytes[:]) + keys += fmt.Sprintf("%s ", addr.String()) + break + } + //blsPubKeyBytes := leaderKey.Object.GetAddress() + //coinbase.SetBytes(blsPubKeyBytes[:]) + + w.Write([]byte(fmt.Sprintf(" %d", s.blockchain.ShardID()))) + w.Write([]byte(fmt.Sprintf(" %s", s.Port))) + w.Write([]byte(fmt.Sprintf(" %s", keys))) + w.Write([]byte(fmt.Sprintf(" %s", s.backend.GetPublicKeys().SerializeToHexStr()))) + +} + +func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { + cur := s.blockchain.CurrentHeader().Number().Uint64() + + for i := cur; i > 0; i-- { + block := s.blockchain.GetBlockByNumber(i) + w.Write([]byte(fmt.Sprintf("%d ", i))) + w.Write([]byte(fmt.Sprintf("%s ", block.Header().ViewID().String()))) + w.Write([]byte(fmt.Sprintf("%s ", block.Header().Coinbase().Hash().Hex()))) + w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) + } +} + +func (s *Service) halt(w http.ResponseWriter, r *http.Request) { + os.Exit(0) +} + // GetHeight returns heights of current and beacon chains if needed. func (s *Service) GetHeight(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index f01cb758ed..0527a0b8c3 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -266,6 +266,8 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { var err error + fmt.Println("OS: ", os.Args) + nodeconfigSetShardSchedule(hc) nodeconfig.SetShardingSchedule(shard.Schedule) nodeconfig.SetVersion(getHarmonyVersion()) @@ -409,8 +411,9 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { if currentNode.NodeConfig.Role() == nodeconfig.Validator { currentNode.RegisterValidatorServices() } else if currentNode.NodeConfig.Role() == nodeconfig.ExplorerNode { - currentNode.RegisterExplorerServices() + } + currentNode.RegisterExplorerServices() currentNode.RegisterService(service.CrosslinkSending, crosslink_sending.New(currentNode, currentNode.Blockchain())) if hc.Pprof.Enabled { setupPprofService(currentNode, hc) @@ -784,6 +787,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // Set the consensus ID to be the current block number viewID := currentNode.Blockchain().CurrentBlock().Header().ViewID().Uint64() + fmt.Println("viewID:", viewID) currentConsensus.SetViewIDs(viewID + 1) utils.Logger().Info(). Uint64("viewID", viewID). diff --git a/consensus/checks.go b/consensus/checks.go index 28da66ad7b..ff4b53927e 100644 --- a/consensus/checks.go +++ b/consensus/checks.go @@ -56,8 +56,7 @@ func (consensus *Consensus) senderKeySanityChecks(msg *msg_pb.Message, senderKey } func (consensus *Consensus) isRightBlockNumAndViewID(recvMsg *FBFTMessage) bool { - blockNum := consensus.getBlockNum() - if recvMsg.ViewID != consensus.getCurBlockViewID() || recvMsg.BlockNum != blockNum { + if recvMsg.ViewID != consensus.GetCurBlockViewID() || recvMsg.BlockNum != consensus.BlockNum() { consensus.getLogger().Debug(). Uint64("blockNum", blockNum). Str("recvMsg", recvMsg.String()). diff --git a/consensus/consensus.go b/consensus/consensus.go index ac15adc15f..69a4f4638f 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -71,6 +71,8 @@ type Consensus struct { priKey multibls.PrivateKeys // the publickey of leader LeaderPubKey *bls.PublicKeyWrapper + // index of leader in the list of validators. + LeaderIndex int // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 0e4bb68141..e646504dde 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,13 +1,13 @@ package consensus import ( + "fmt" "math/big" "sync/atomic" "time" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/crypto/bls" - "github.com/harmony-one/harmony/multibls" "github.com/ethereum/go-ethereum/common" protobuf "github.com/golang/protobuf/proto" @@ -74,12 +74,8 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - return consensus.updatePublicKeys(pubKeys, allowlist) -} - -func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { + // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. + consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) consensus.getLogger().Info().Msg("My Committee updated") for i := range pubKeys { @@ -98,20 +94,15 @@ func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.Publi consensus.getLogger().Error(). Msg("[UpdatePublicKeys] Participants is empty") } - for i := range pubKeys { - consensus.getLogger().Info(). - Int("index", i). - Str("BLSPubKey", pubKeys[i].Bytes.Hex()). - Msg("Member") - } + consensus.pubKeyLock.Unlock() // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. - consensus.updateBitmaps() - consensus.resetState() + consensus.UpdateBitmaps() + consensus.ResetState() // do not reset view change state if it is in view changing mode - if !consensus.isViewChangingMode() { - consensus.resetViewChangeState() + if !consensus.IsViewChangingMode() { + consensus.ResetViewChangeState() } return consensus.Decider.ParticipantsCount() } @@ -138,7 +129,7 @@ func (consensus *Consensus) signConsensusMessage(message *msg_pb.Message, } // UpdateBitmaps update the bitmaps for prepare and commit phase -func (consensus *Consensus) updateBitmaps() { +func (consensus *Consensus) UpdateBitmaps() { consensus.getLogger().Debug(). Str("MessageType", consensus.phase.String()). Msg("[UpdateBitmaps] Updating consensus bitmaps") @@ -148,12 +139,13 @@ func (consensus *Consensus) updateBitmaps() { multiSigBitmap, _ := bls_cosi.NewMask(members, nil) consensus.prepareBitmap = prepareBitmap consensus.commitBitmap = commitBitmap + consensus.multiSigMutex.Lock() consensus.multiSigBitmap = multiSigBitmap - + consensus.multiSigMutex.Unlock() } // ResetState resets the state of the consensus -func (consensus *Consensus) resetState() { +func (consensus *Consensus) ResetState() { consensus.switchPhase("ResetState", FBFTAnnounce) consensus.blockHash = [32]byte{} @@ -171,24 +163,11 @@ func (consensus *Consensus) resetState() { // IsValidatorInCommittee returns whether the given validator BLS address is part of my committee func (consensus *Consensus) IsValidatorInCommittee(pubKey bls.SerializedPublicKey) bool { - consensus.mutex.RLock() - defer consensus.mutex.RUnlock() - return consensus.isValidatorInCommittee(pubKey) -} - -func (consensus *Consensus) isValidatorInCommittee(pubKey bls.SerializedPublicKey) bool { return consensus.Decider.IndexOf(pubKey) != -1 } // SetMode sets the mode of consensus func (consensus *Consensus) SetMode(m Mode) { - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - consensus.setMode(m) -} - -// SetMode sets the mode of consensus -func (consensus *Consensus) setMode(m Mode) { if m == Normal && consensus.isBackup { m = NormalBackup } @@ -210,13 +189,6 @@ func (consensus *Consensus) SetIsBackup(isBackup bool) { // Mode returns the mode of consensus func (consensus *Consensus) Mode() Mode { - consensus.mutex.RLock() - defer consensus.mutex.RUnlock() - return consensus.mode() -} - -// mode returns the mode of consensus -func (consensus *Consensus) mode() Mode { return consensus.current.Mode() } @@ -236,11 +208,12 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if consensus.IgnoreViewIDCheck.IsSet() { //in syncing mode, node accepts incoming messages without viewID/leaderKey checking //so only set mode to normal when new node enters consensus and need checking viewID - consensus.setMode(Normal) - consensus.setViewIDs(msg.ViewID) + consensus.SetMode(Normal) + consensus.SetViewIDs(msg.ViewID) if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } + fmt.Println("[checkViewID] Set LEADEER PUB KEY ", msg.SenderPubkeys[0].Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() @@ -248,9 +221,9 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { Str("leaderKey", consensus.LeaderPubKey.Bytes.Hex()). Msg("[checkViewID] Start consensus timer") return nil - } else if msg.ViewID > consensus.getCurBlockViewID() { + } else if msg.ViewID > consensus.GetCurBlockViewID() { return consensus_engine.ErrViewIDNotMatch - } else if msg.ViewID < consensus.getCurBlockViewID() { + } else if msg.ViewID < consensus.GetCurBlockViewID() { return errors.New("view ID belongs to the past") } return nil @@ -261,26 +234,17 @@ func (consensus *Consensus) SetBlockNum(blockNum uint64) { atomic.StoreUint64(&consensus.blockNum, blockNum) } -// SetBlockNum sets the blockNum in consensus object, called at node bootstrap -func (consensus *Consensus) setBlockNum(blockNum uint64) { - atomic.StoreUint64(&consensus.blockNum, blockNum) -} - // ReadSignatureBitmapPayload read the payload for signature and bitmap; offset is the beginning position of reading -func (consensus *Consensus) ReadSignatureBitmapPayload(recvPayload []byte, offset int) (*bls_core.Sign, *bls_cosi.Mask, error) { - consensus.mutex.RLock() - members := consensus.Decider.Participants() - consensus.mutex.RUnlock() - return consensus.readSignatureBitmapPayload(recvPayload, offset, members) -} - -func (consensus *Consensus) readSignatureBitmapPayload(recvPayload []byte, offset int, members multibls.PublicKeys) (*bls_core.Sign, *bls_cosi.Mask, error) { +func (consensus *Consensus) ReadSignatureBitmapPayload( + recvPayload []byte, offset int, +) (*bls_core.Sign, *bls_cosi.Mask, error) { if offset+bls.BLSSignatureSizeInBytes > len(recvPayload) { return nil, nil, errors.New("payload not have enough length") } sigAndBitmapPayload := recvPayload[offset:] // TODO(audit): keep a Mask in the Decider so it won't be reconstructed on the fly. + members := consensus.Decider.Participants() return chain.ReadSignatureBitmapByPublicKeys( sigAndBitmapPayload, members, ) @@ -297,12 +261,6 @@ func (consensus *Consensus) readSignatureBitmapPayload(recvPayload []byte, offse // (b) node in committed but has any err during processing: Syncing mode // (c) node in committed and everything looks good: Normal mode func (consensus *Consensus) UpdateConsensusInformation() Mode { - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - return consensus.updateConsensusInformation() -} - -func (consensus *Consensus) updateConsensusInformation() Mode { curHeader := consensus.Blockchain().CurrentHeader() curEpoch := curHeader.Epoch() nextEpoch := new(big.Int).Add(curHeader.Epoch(), common.Big1) @@ -407,7 +365,7 @@ func (consensus *Consensus) updateConsensusInformation() Mode { consensus.getLogger().Info(). Int("numPubKeys", len(pubKeys)). Msg("[UpdateConsensusInformation] Successfully updated public keys") - consensus.updatePublicKeys(pubKeys, shard.Schedule.InstanceForEpoch(nextEpoch).ExternalAllowlist()) + consensus.UpdatePublicKeys(pubKeys, shard.Schedule.InstanceForEpoch(nextEpoch).ExternalAllowlist()) // Update voters in the committee if _, err := consensus.Decider.SetVoters( @@ -438,13 +396,16 @@ func (consensus *Consensus) updateConsensusInformation() Mode { consensus.getLogger().Info(). Str("leaderPubKey", leaderPubKey.Bytes.Hex()). Msg("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain") + consensus.pubKeyLock.Lock() + fmt.Println("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain", leaderPubKey.Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = leaderPubKey + consensus.pubKeyLock.Unlock() } } for _, key := range pubKeys { // in committee - myPubKeys := consensus.getPublicKeys() + myPubKeys := consensus.GetPublicKeys() if myPubKeys.Contains(key.Object) { if hasError { consensus.getLogger().Error(). @@ -456,7 +417,7 @@ func (consensus *Consensus) updateConsensusInformation() Mode { // If the leader changed and I myself become the leader if (oldLeader != nil && consensus.LeaderPubKey != nil && - !consensus.LeaderPubKey.Object.IsEqual(oldLeader.Object)) && consensus.isLeader() { + !consensus.LeaderPubKey.Object.IsEqual(oldLeader.Object)) && consensus.IsLeader() { go func() { consensus.getLogger().Info(). Str("myKey", myPubKeys.SerializeToHexStr()). @@ -477,10 +438,15 @@ func (consensus *Consensus) updateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { - consensus.mutex.RLock() - defer consensus.mutex.RUnlock() - - return consensus.isLeader() + consensus.pubKeyLock.Lock() + obj := consensus.LeaderPubKey.Object + consensus.pubKeyLock.Unlock() + for _, key := range consensus.priKey { + if key.Pub.Object.IsEqual(obj) { + return true + } + } + return false } // isLeader check if the node is a leader or not by comparing the public key of @@ -498,16 +464,9 @@ func (consensus *Consensus) isLeader() bool { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - consensus.setViewIDs(height) -} - -// SetViewIDs set both current view ID and view changing ID to the height -// of the blockchain. It is used during client startup to recover the state -func (consensus *Consensus) setViewIDs(height uint64) { - consensus.setCurBlockViewID(height) - consensus.setViewChangingID(height) + fmt.Println("SetViewIDs", height) + consensus.SetCurBlockViewID(height) + consensus.SetViewChangingID(height) } // SetCurBlockViewID set the current view ID @@ -515,18 +474,22 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } -// SetCurBlockViewID set the current view ID -func (consensus *Consensus) setCurBlockViewID(viewID uint64) { - consensus.current.SetCurBlockViewID(viewID) +// SetLeaderIndex set the leader index. +func (consensus *Consensus) SetLeaderIndex(f func(int) int) (current int) { + consensus.pubKeyLock.Lock() + defer consensus.pubKeyLock.Unlock() + consensus.LeaderIndex = f(consensus.LeaderIndex) + return consensus.LeaderIndex } -// SetViewChangingID set the current view change ID -func (consensus *Consensus) SetViewChangingID(viewID uint64) { - consensus.current.SetViewChangingID(viewID) +func (consensus *Consensus) GetLeaderIndex() int { + consensus.pubKeyLock.Lock() + defer consensus.pubKeyLock.Unlock() + return consensus.LeaderIndex } // SetViewChangingID set the current view change ID -func (consensus *Consensus) setViewChangingID(viewID uint64) { +func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) } @@ -535,6 +498,15 @@ func (consensus *Consensus) StartFinalityCount() { consensus.finalityCounter.Store(time.Now().UnixNano()) } +//func (consensus *Consensus) ReshardingNextLeader(newblock *types.Block) { +// consensus.pubKeyLock.Lock() +// fmt.Println("nextBlock1 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) +// consensus.LeaderPubKey = consensus.getNextLeaderKey(consensus.GetCurBlockViewID() + 1) +// fmt.Println("nextBlock2 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) +// consensus.pubKeyLock.Unlock() +// +//} + // FinishFinalityCount calculate the current finality func (consensus *Consensus) FinishFinalityCount() { d := time.Now().UnixNano() @@ -554,7 +526,8 @@ func (consensus *Consensus) switchPhase(subject string, desired FBFTPhase) { Str("to:", desired.String()). Str("switchPhase:", subject) - consensus.phase = desired + consensus.phase.Set(desired) + return } var ( @@ -576,13 +549,13 @@ func (consensus *Consensus) selfCommit(payload []byte) error { return errGetPreparedBlock } - aggSig, mask, err := consensus.readSignatureBitmapPayload(payload, 32, consensus.Decider.Participants()) + aggSig, mask, err := consensus.ReadSignatureBitmapPayload(payload, 32) if err != nil { return errReadBitmapPayload } // Have to keep the block hash so the leader can finish the commit phase of prepared block - consensus.resetState() + consensus.ResetState() copy(consensus.blockHash[:], blockHash[:]) consensus.switchPhase("selfCommit", FBFTCommit) @@ -639,18 +612,11 @@ func (consensus *Consensus) NumSignaturesIncludedInBlock(block *types.Block) uin return count } -// GetLogger returns logger for consensus contexts added. -func (consensus *Consensus) GetLogger() *zerolog.Logger { - consensus.mutex.RLock() - defer consensus.mutex.RUnlock() - return consensus.getLogger() -} - // getLogger returns logger for consensus contexts added func (consensus *Consensus) getLogger() *zerolog.Logger { logger := utils.Logger().With(). - Uint64("myBlock", consensus.blockNum). - Uint64("myViewID", consensus.getCurBlockViewID()). + Uint64("myBlock", consensus.BlockNum()). + Uint64("myViewID", consensus.GetCurBlockViewID()). Str("phase", consensus.phase.String()). Str("mode", consensus.current.Mode().String()). Logger() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 99bccf7554..16554d0393 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -4,13 +4,13 @@ import ( "bytes" "context" "encoding/hex" - "math/big" + "fmt" "sync/atomic" "time" - "github.com/ethereum/go-ethereum/common" bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" + nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" @@ -46,24 +46,16 @@ const ( // IsViewChangingMode return true if curernt mode is viewchanging func (consensus *Consensus) IsViewChangingMode() bool { - consensus.mutex.RLock() - defer consensus.mutex.RUnlock() - return consensus.isViewChangingMode() -} - -func (consensus *Consensus) isViewChangingMode() bool { return consensus.current.Mode() == ViewChanging } // HandleMessageUpdate will update the consensus state according to received message func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb.Message, senderKey *bls.SerializedPublicKey) error { - consensus.mutex.Lock() - defer consensus.mutex.Unlock() // when node is in ViewChanging mode, it still accepts normal messages into FBFTLog // in order to avoid possible trap forever but drop PREPARE and COMMIT // which are message types specifically for a node acting as leader // so we just ignore those messages - if consensus.isViewChangingMode() && + if consensus.IsViewChangingMode() && (msg.Type == msg_pb.MessageType_PREPARE || msg.Type == msg_pb.MessageType_COMMIT) { return nil @@ -95,7 +87,7 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb members := consensus.Decider.Participants() fbftMsg, err = ParseNewViewMessage(msg, members) default: - fbftMsg, err = consensus.parseFBFTMessage(msg) + fbftMsg, err = consensus.ParseFBFTMessage(msg) } if err != nil || fbftMsg == nil { return errors.Wrapf(err, "unable to parse consensus msg with type: %s", msg.Type) @@ -103,8 +95,8 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb canHandleViewChange := true intendedForValidator, intendedForLeader := - !consensus.isLeader(), - consensus.isLeader() + !consensus.IsLeader(), + consensus.IsLeader() // if in backup normal mode, force ignore view change event and leader event. if consensus.current.Mode() == NormalBackup { @@ -139,14 +131,15 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb } func (consensus *Consensus) finalCommit() { + // THIS IS NOT GOOD PLACE FOR LEADER SWITCHING numCommits := consensus.Decider.SignersCount(quorum.Commit) consensus.getLogger().Info(). Int64("NumCommits", numCommits). Msg("[finalCommit] Finalizing Consensus") - beforeCatchupNum := consensus.getBlockNum() + beforeCatchupNum := consensus.BlockNum() - leaderPriKey, err := consensus.getConsensusLeaderPrivateKey() + leaderPriKey, err := consensus.GetConsensusLeaderPrivateKey() if err != nil { consensus.getLogger().Error().Err(err).Msg("[finalCommit] leader not found") return @@ -185,7 +178,7 @@ func (consensus *Consensus) finalCommit() { // Note: leader already sent 67% commit in preCommit. The 100% commit won't be sent immediately // to save network traffic. It will only be sent in retry if consensus doesn't move forward. // Or if the leader is changed for next block, the 100% committed sig will be sent to the next leader immediately. - if !consensus.isLeader() || block.IsLastBlockInEpoch() { + if !consensus.IsLeader() || block.IsLastBlockInEpoch() { // send immediately if err := consensus.msgSender.SendWithRetry( block.NumberU64(), @@ -250,7 +243,7 @@ func (consensus *Consensus) finalCommit() { // If still the leader, send commit sig/bitmap to finish the new block proposal, // else, the block proposal will timeout by itself. - if consensus.isLeader() { + if consensus.IsLeader() { if block.IsLastBlockInEpoch() { // No pipelining go func() { @@ -298,29 +291,143 @@ func (consensus *Consensus) BlockCommitSigs(blockNum uint64) ([]byte, error) { // Start waits for the next new block and run consensus func (consensus *Consensus) Start( - stopChan chan struct{}, + blockChannel chan *types.Block, stopChan, stoppedChan, startChannel chan struct{}, ) { go func() { + toStart := make(chan struct{}, 1) + isInitialLeader := consensus.IsLeader() + if isInitialLeader { + consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Waiting for consensus start") + // send a signal to indicate it's ready to run consensus + // this signal is consumed by node object to create a new block and in turn trigger a new consensus on it + go func() { + <-startChannel + toStart <- struct{}{} + consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Send ReadySignal") + consensus.ReadySignal <- SyncProposal + }() + } consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Consensus started") - go func() { - ticker := time.NewTicker(250 * time.Millisecond) - defer ticker.Stop() - for { - select { - case <-stopChan: - return - case <-ticker.C: - consensus.Tick() - } - } - }() - - consensus.mutex.Lock() + defer close(stoppedChan) + ticker := time.NewTicker(250 * time.Millisecond) + defer ticker.Stop() consensus.consensusTimeout[timeoutBootstrap].Start() consensus.getLogger().Info().Msg("[ConsensusMainLoop] Start bootstrap timeout (only once)") + // Set up next block due time. consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) - consensus.mutex.Unlock() + start := false + for { + select { + case <-toStart: + start = true + case <-ticker.C: + if !start && isInitialLeader { + continue + } + for k, v := range consensus.consensusTimeout { + // stop timer in listening mode + if consensus.current.Mode() == Listening { + v.Stop() + continue + } + + if consensus.current.Mode() == Syncing { + // never stop bootstrap timer here in syncing mode as it only starts once + // if it is stopped, bootstrap will be stopped and nodes + // can't start view change or join consensus + // the bootstrap timer will be stopped once consensus is reached or view change + // is succeeded + if k != timeoutBootstrap { + consensus.getLogger().Debug(). + Str("k", k.String()). + Str("Mode", consensus.current.Mode().String()). + Msg("[ConsensusMainLoop] consensusTimeout stopped!!!") + v.Stop() + continue + } + } + if !v.CheckExpire() { + continue + } + if k != timeoutViewChange { + consensus.getLogger().Warn().Msg("[ConsensusMainLoop] Ops Consensus Timeout!!!") + consensus.startViewChange() + break + } else { + consensus.getLogger().Warn().Msg("[ConsensusMainLoop] Ops View Change Timeout!!!") + consensus.startViewChange() + break + } + } + + // TODO: Refactor this piece of code to consensus/downloader.go after DNS legacy sync is removed + case <-consensus.syncReadyChan: + consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncReadyChan") + consensus.mutex.Lock() + if consensus.BlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { + consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) + consensus.SetViewIDs(consensus.Blockchain().CurrentHeader().ViewID().Uint64() + 1) + mode := consensus.UpdateConsensusInformation() + consensus.current.SetMode(mode) + consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") + consensus.consensusTimeout[timeoutConsensus].Start() + consensus.getLogger().Info().Str("Mode", mode.String()).Msg("Node is IN SYNC") + consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() + } else if consensus.Mode() == Syncing { + // Corner case where sync is triggered before `onCommitted` and there is a race + // for block insertion between consensus and downloader. + mode := consensus.UpdateConsensusInformation() + consensus.SetMode(mode) + consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") + consensus.consensusTimeout[timeoutConsensus].Start() + consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() + } + consensus.mutex.Unlock() + + // TODO: Refactor this piece of code to consensus/downloader.go after DNS legacy sync is removed + case <-consensus.syncNotReadyChan: + consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncNotReadyChan") + consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) + consensus.current.SetMode(Syncing) + consensus.getLogger().Info().Msg("[ConsensusMainLoop] Node is OUT OF SYNC") + consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() + + case newBlock := <-blockChannel: + //consensus.ReshardingNextLeader(newBlock) + consensus.getLogger().Info(). + Uint64("MsgBlockNum", newBlock.NumberU64()). + Msg("[ConsensusMainLoop] Received Proposed New Block!") + + if newBlock.NumberU64() < consensus.BlockNum() { + consensus.getLogger().Warn().Uint64("newBlockNum", newBlock.NumberU64()). + Msg("[ConsensusMainLoop] received old block, abort") + continue + } + // Sleep to wait for the full block time + consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") + + <-time.After(time.Until(consensus.NextBlockDue)) + consensus.StartFinalityCount() + + // Update time due for next block + consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) + + startTime = time.Now() + consensus.msgSender.Reset(newBlock.NumberU64()) + + consensus.getLogger().Info(). + Int("numTxs", len(newBlock.Transactions())). + Int("numStakingTxs", len(newBlock.StakingTransactions())). + Time("startTime", startTime). + Int64("publicKeys", consensus.Decider.ParticipantsCount()). + Msg("[ConsensusMainLoop] STARTING CONSENSUS") + consensus.announce(newBlock) + case <-stopChan: + consensus.getLogger().Info().Msg("[ConsensusMainLoop] stopChan") + return + } + } }() if consensus.dHelper != nil { @@ -328,129 +435,28 @@ func (consensus *Consensus) Start( } } -func (consensus *Consensus) StartChannel() { - consensus.mutex.Lock() - consensus.isInitialLeader = consensus.isLeader() - if consensus.isInitialLeader { - consensus.start = true - consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Send ReadySignal") - consensus.mutex.Unlock() - consensus.ReadySignal <- SyncProposal - return - } - consensus.mutex.Unlock() -} - -func (consensus *Consensus) syncReadyChan() { - consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncReadyChan") - if consensus.getBlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { - consensus.setBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) - consensus.setViewIDs(consensus.Blockchain().CurrentHeader().ViewID().Uint64() + 1) - mode := consensus.updateConsensusInformation() - consensus.current.SetMode(mode) - consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") - consensus.consensusTimeout[timeoutConsensus].Start() - consensus.getLogger().Info().Str("Mode", mode.String()).Msg("Node is IN SYNC") - consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() - } else if consensus.mode() == Syncing { - // Corner case where sync is triggered before `onCommitted` and there is a race - // for block insertion between consensus and downloader. - mode := consensus.updateConsensusInformation() - consensus.setMode(mode) - consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") - consensus.consensusTimeout[timeoutConsensus].Start() - consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() - } -} - -func (consensus *Consensus) syncNotReadyChan() { - consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncNotReadyChan") - consensus.setBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) - consensus.current.SetMode(Syncing) - consensus.getLogger().Info().Msg("[ConsensusMainLoop] Node is OUT OF SYNC") - consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() -} - -func (consensus *Consensus) Tick() { - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - consensus.tick() -} - -func (consensus *Consensus) tick() { - if !consensus.start && consensus.isInitialLeader { - return - } - for k, v := range consensus.consensusTimeout { - // stop timer in listening mode - if consensus.current.Mode() == Listening { - v.Stop() - continue - } - - if consensus.current.Mode() == Syncing { - // never stop bootstrap timer here in syncing mode as it only starts once - // if it is stopped, bootstrap will be stopped and nodes - // can't start view change or join consensus - // the bootstrap timer will be stopped once consensus is reached or view change - // is succeeded - if k != timeoutBootstrap { - consensus.getLogger().Debug(). - Str("k", k.String()). - Str("Mode", consensus.current.Mode().String()). - Msg("[ConsensusMainLoop] consensusTimeout stopped!!!") - v.Stop() - continue - } - } - if !v.CheckExpire() { - continue - } - if k != timeoutViewChange { - consensus.getLogger().Warn().Msg("[ConsensusMainLoop] Ops Consensus Timeout!!!") - consensus.startViewChange() - break - } else { - consensus.getLogger().Warn().Msg("[ConsensusMainLoop] Ops View Change Timeout!!!") - consensus.startViewChange() - break - } +// Close closes the consensus. If current is in normal commit phase, wait until the commit +// phase end. +func (consensus *Consensus) Close() error { + if consensus.dHelper != nil { + consensus.dHelper.close() } + consensus.waitForCommit() + return nil } -func (consensus *Consensus) BlockChannel(newBlock *types.Block) { - consensus.GetLogger().Info(). - Uint64("MsgBlockNum", newBlock.NumberU64()). - Msg("[ConsensusMainLoop] Received Proposed New Block!") - - if newBlock.NumberU64() < consensus.BlockNum() { - consensus.getLogger().Warn().Uint64("newBlockNum", newBlock.NumberU64()). - Msg("[ConsensusMainLoop] received old block, abort") +// waitForCommit wait extra 2 seconds for commit phase to finish +func (consensus *Consensus) waitForCommit() { + if consensus.Mode() != Normal || consensus.phase.Get() != FBFTCommit { return } - // Sleep to wait for the full block time - consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") - time.AfterFunc(time.Until(consensus.NextBlockDue), func() { - consensus.StartFinalityCount() - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - // Update time due for next block - consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) - - startTime = time.Now() - consensus.msgSender.Reset(newBlock.NumberU64()) + // We only need to wait consensus is in normal commit phase + utils.Logger().Warn().Str("phase", consensus.phase.String()).Msg("[shutdown] commit phase has to wait") - consensus.getLogger().Info(). - Int("numTxs", len(newBlock.Transactions())). - Int("numStakingTxs", len(newBlock.StakingTransactions())). - Time("startTime", startTime). - Int64("publicKeys", consensus.Decider.ParticipantsCount()). - Msg("[ConsensusMainLoop] STARTING CONSENSUS") - consensus.announce(newBlock) - }) - - if consensus.dHelper != nil { - consensus.dHelper.start() + maxWait := time.Now().Add(2 * consensus.BlockPeriod) + for time.Now().Before(maxWait) && consensus.GetConsensusPhase() == "Commit" { + utils.Logger().Warn().Msg("[shutdown] wait for consensus finished") + time.Sleep(time.Millisecond * 100) } } @@ -465,29 +471,24 @@ type LastMileBlockIter struct { } // GetLastMileBlockIter get the iterator of the last mile blocks starting from number bnStart -func (consensus *Consensus) GetLastMileBlockIter(bnStart uint64, cb func(iter *LastMileBlockIter) error) error { +func (consensus *Consensus) GetLastMileBlockIter(bnStart uint64) (*LastMileBlockIter, error) { consensus.mutex.Lock() defer consensus.mutex.Unlock() - return consensus.getLastMileBlockIter(bnStart, cb) -} - -// GetLastMileBlockIter get the iterator of the last mile blocks starting from number bnStart -func (consensus *Consensus) getLastMileBlockIter(bnStart uint64, cb func(iter *LastMileBlockIter) error) error { if consensus.BlockVerifier == nil { - return errors.New("consensus haven't initialized yet") + return nil, errors.New("consensus haven't initialized yet") } blocks, _, err := consensus.getLastMileBlocksAndMsg(bnStart) if err != nil { - return err + return nil, err } - return cb(&LastMileBlockIter{ + return &LastMileBlockIter{ blockCandidates: blocks, fbftLog: consensus.FBFTLog, verify: consensus.BlockVerifier, curIndex: 0, logger: consensus.getLogger(), - }) + }, nil } // Next iterate to the next last mile block @@ -530,11 +531,12 @@ func (consensus *Consensus) getLastMileBlocksAndMsg(bnStart uint64) ([]*types.Bl // preCommitAndPropose commit the current block with 67% commit signatures and start // proposing new block which will wait on the full commit signatures to finish func (consensus *Consensus) preCommitAndPropose(blk *types.Block) error { + //fmt.Println("preCommitAndPropose", utils.GetPort(), blk.NumberU64()) if blk == nil { return errors.New("block to pre-commit is nil") } - leaderPriKey, err := consensus.getConsensusLeaderPrivateKey() + leaderPriKey, err := consensus.GetConsensusLeaderPrivateKey() if err != nil { consensus.getLogger().Error().Err(err).Msg("[preCommitAndPropose] leader not found") return err @@ -624,7 +626,7 @@ func (consensus *Consensus) tryCatchup() error { if consensus.BlockVerifier == nil { return errors.New("consensus haven't finished initialization") } - initBN := consensus.getBlockNum() + initBN := consensus.BlockNum() defer consensus.postCatchup(initBN) blks, msgs, err := consensus.getLastMileBlocksAndMsg(initBN) @@ -638,7 +640,7 @@ func (consensus *Consensus) tryCatchup() error { } blk.SetCurrentCommitSig(msg.Payload) - if err := consensus.verifyBlock(blk); err != nil { + if err := consensus.VerifyBlock(blk); err != nil { consensus.getLogger().Err(err).Msg("[TryCatchup] failed block verifier") return err } @@ -647,6 +649,7 @@ func (consensus *Consensus) tryCatchup() error { consensus.getLogger().Error().Err(err).Msg("[TryCatchup] Failed to add block to chain") return err } + //fmt.Println("tryCatchup ", utils.GetPort(), blk.NumberU64()) select { // TODO: Remove this when removing dns sync and stream sync is fully up case consensus.VerifiedNewBlock <- blk: @@ -661,6 +664,8 @@ func (consensus *Consensus) tryCatchup() error { } func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMessage) error { + // this function evaluates for all, leader and validators. + if consensus.Blockchain().CurrentBlock().NumberU64() < blk.NumberU64() { if _, err := consensus.Blockchain().InsertChain([]*types.Block{blk}, !consensus.FBFTLog.IsBlockVerified(blk.Hash())); err != nil { consensus.getLogger().Error().Err(err).Msg("[commitBlock] Failed to add block to chain") @@ -674,102 +679,58 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess } consensus.FinishFinalityCount() - go func() { - consensus.PostConsensusJob(blk) - }() - consensus.setupForNewConsensus(blk, committedMsg) + consensus.PostConsensusJob(blk) + consensus.SetupForNewConsensus(blk, committedMsg) utils.Logger().Info().Uint64("blockNum", blk.NumberU64()). Str("hash", blk.Header().Hash().Hex()). Msg("Added New Block to Blockchain!!!") - return nil } -// rotateLeader rotates the leader to the next leader in the committee. -// This function must be called with enabled leader rotation. -func (consensus *Consensus) rotateLeader(epoch *big.Int) { - prev := consensus.getLeaderPubKey() - bc := consensus.Blockchain() - curNumber := bc.CurrentHeader().Number().Uint64() - utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), bc.Config().IsLeaderRotation(epoch), bc.Config().LeaderRotationBlocksCount) - leader := consensus.getLeaderPubKey() - for i := 0; i < bc.Config().LeaderRotationBlocksCount; i++ { - header := bc.GetHeaderByNumber(curNumber - uint64(i)) - if header == nil { - return - } - // Previous epoch, we should not change leader. - if header.Epoch().Uint64() != epoch.Uint64() { - return - } - // Check if the same leader. - pub, err := bc.GetLeaderPubKeyFromCoinbase(header) - if err != nil { - utils.Logger().Error().Err(err).Msg("Failed to get leader public key from coinbase") - return - } - if !pub.Object.IsEqual(leader.Object) { - // Another leader. - return +// SetupForNewConsensus sets the state for new consensus +func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { + atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) + curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) // first view id is going to be 2. + prev := consensus.GetLeaderPubKey() + idx := consensus.SetLeaderIndex(func(i int) int { + if curBlockViewID%3 == 0 { + return i + 1 } - } - // Passed all checks, we can change leader. - var ( - wasFound bool - next *bls.PublicKeyWrapper - ) - if consensus.ShardID == shard.BeaconChainShardID { - wasFound, next = consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) - } else { - wasFound, next = consensus.Decider.NthNext(leader, 1) - } - if !wasFound { - utils.Logger().Error().Msg("Failed to get next leader") - return - } else { - consensus.setLeaderPubKey(next) - } - if consensus.isLeader() && !consensus.getLeaderPubKey().Object.IsEqual(prev.Object) { + return i + }) + pps := consensus.Decider.Participants() + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = &pps[idx%len(pps)] + fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %q\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + consensus.pubKeyLock.Unlock() + if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { // leader changed go func() { + fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) consensus.ReadySignal <- SyncProposal + }() - } -} -// SetupForNewConsensus sets the state for new consensus -func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { - atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) - consensus.setCurBlockViewID(committedMsg.ViewID + 1) - consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] - var epoch *big.Int - if blk.IsLastBlockInEpoch() { - epoch = new(big.Int).Add(blk.Epoch(), common.Big1) - } else { - epoch = blk.Epoch() } - if consensus.Blockchain().Config().IsLeaderRotation(epoch) { - consensus.rotateLeader(epoch) - } - // Update consensus keys at last so the change of leader status doesn't mess up normal flow if blk.IsLastBlockInEpoch() { - consensus.setMode(consensus.updateConsensusInformation()) + consensus.SetMode(consensus.UpdateConsensusInformation()) } consensus.FBFTLog.PruneCacheBeforeBlock(blk.NumberU64()) - consensus.resetState() + consensus.ResetState() } func (consensus *Consensus) postCatchup(initBN uint64) { - if initBN < consensus.getBlockNum() { + if initBN < consensus.BlockNum() { consensus.getLogger().Info(). Uint64("From", initBN). - Uint64("To", consensus.getBlockNum()). + Uint64("To", consensus.BlockNum()). Msg("[TryCatchup] Caught up!") consensus.switchPhase("TryCatchup", FBFTAnnounce) } // catch up and skip from view change trap - if initBN < consensus.getBlockNum() && consensus.isViewChangingMode() { + if initBN < consensus.BlockNum() && consensus.IsViewChangingMode() { consensus.current.SetMode(Normal) consensus.consensusTimeout[timeoutViewChange].Stop() } @@ -777,7 +738,7 @@ func (consensus *Consensus) postCatchup(initBN uint64) { // GenerateVrfAndProof generates new VRF/Proof from hash of previous block func (consensus *Consensus) GenerateVrfAndProof(newHeader *block.Header) error { - key, err := consensus.getConsensusLeaderPrivateKey() + key, err := consensus.GetConsensusLeaderPrivateKey() if err != nil { return errors.New("[GenerateVrfAndProof] no leader private key provided") } @@ -830,7 +791,7 @@ func (consensus *Consensus) GenerateVdfAndProof(newBlock *types.Block, vrfBlockN start := time.Now() vdf.Execute() duration := time.Since(start) - consensus.GetLogger().Info(). + consensus.getLogger().Info(). Dur("duration", duration). Msg("[ConsensusMainLoop] VDF computation finished") output := <-outputChannel diff --git a/consensus/leader.go b/consensus/leader.go index 2f7766e19b..21f3b770d5 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -190,7 +190,17 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { + // TODO HERE + //if recvMsg.ViewID == 10 { + // return + //} + consensus.mutex.Lock() + defer consensus.mutex.Unlock() //// Read - Start + if consensus.ShardID == 0 { + //fmt.Println("onCommit ", recvMsg.BlockNum) + } + if !consensus.isRightBlockNumAndViewID(recvMsg) { return } @@ -322,4 +332,5 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED) } + //fmt.Println("onCommit99: ", utils.GetPort(), recvMsg.BlockNum) } diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index aba62fa53f..52b765a88c 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -230,12 +230,17 @@ func (s *cIdentities) NthNextHmy(instance shardingconfig.Instance, pubKey *bls.P Msg("[NthNextHmy] pubKey not found") } numNodes := instance.NumHarmonyOperatedNodesPerShard() + //fmt.Println("??idx:", idx, numNodes) // sanity check to avoid out of bound access if numNodes <= 0 || numNodes > len(s.publicKeys) { numNodes = len(s.publicKeys) } idx = (idx + next) % numNodes - return found, &s.publicKeys[idx] + //fmt.Println("-------idx:", idx) + + new := &s.publicKeys[idx] + fmt.Println("NthNextHmy: ", pubKey.Bytes.Hex(), new.Bytes.Hex()) + return found, new } // NthNextHmyExt return the Nth next pubkey of Harmony + allowlist nodes, next can be negative number diff --git a/consensus/validator.go b/consensus/validator.go index f85cb8e3d8..013769395c 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -18,7 +18,8 @@ import ( ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { - recvMsg, err := consensus.parseFBFTMessage(msg) + + recvMsg, err := consensus.ParseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). Err(err). @@ -26,6 +27,9 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Msg("[OnAnnounce] Unparseable leader message") return } + if consensus.ShardID == 0 { + //fmt.Println("onAnnounce called ", recvMsg.BlockNum) + } // NOTE let it handle its own logs if !consensus.onAnnounceSanityChecks(recvMsg) { @@ -38,10 +42,12 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Uint64("MsgBlockNum", recvMsg.BlockNum). Msg("[OnAnnounce] Announce message Added") consensus.FBFTLog.AddVerifiedMessage(recvMsg) + consensus.mutex.Lock() + defer consensus.mutex.Unlock() consensus.blockHash = recvMsg.BlockHash // we have already added message and block, skip check viewID // and send prepare message if is in ViewChanging mode - if consensus.isViewChangingMode() { + if consensus.IsViewChangingMode() { consensus.getLogger().Debug(). Msg("[OnAnnounce] Still in ViewChanging Mode, Exiting !!") return @@ -62,7 +68,7 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { if len(recvMsg.Block) > 0 { go func() { // Best effort check, no need to error out. - _, err := consensus.ValidateNewBlock(recvMsg) + _, err := consensus.validateNewBlock(recvMsg) if err == nil { consensus.getLogger().Info(). @@ -72,12 +78,11 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { } } -func (consensus *Consensus) ValidateNewBlock(recvMsg *FBFTMessage) (*types.Block, error) { - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - return consensus.validateNewBlock(recvMsg) -} func (consensus *Consensus) validateNewBlock(recvMsg *FBFTMessage) (*types.Block, error) { + // Lock to prevent race condition between announce and prepare + consensus.verifyBlockMutex.Lock() + defer consensus.verifyBlockMutex.Unlock() + if consensus.FBFTLog.IsBlockVerified(recvMsg.BlockHash) { var blockObj *types.Block @@ -131,7 +136,7 @@ func (consensus *Consensus) validateNewBlock(recvMsg *FBFTMessage) (*types.Block return nil, errors.New("nil block verifier") } - if err := consensus.verifyBlock(&blockObj); err != nil { + if err := consensus.VerifyBlock(&blockObj); err != nil { consensus.getLogger().Error().Err(err).Msg("[validateNewBlock] Block verification failed") return nil, errors.New("Block verification failed") } @@ -183,6 +188,12 @@ func (consensus *Consensus) sendCommitMessages(blockObj *types.Block) { // if onPrepared accepts the prepared message from the leader, then // it will send a COMMIT message for the leader to receive on the network. func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { + if consensus.ShardID == 0 { + //fmt.Println("onPrepared", recvMsg.BlockNum) + } + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + consensus.getLogger().Info(). Uint64("MsgBlockNum", recvMsg.BlockNum). Uint64("MsgViewID", recvMsg.ViewID). @@ -205,7 +216,7 @@ func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { // check validity of prepared signature blockHash := recvMsg.BlockHash - aggSig, mask, err := consensus.readSignatureBitmapPayload(recvMsg.Payload, 0, consensus.Decider.Participants()) + aggSig, mask, err := consensus.ReadSignatureBitmapPayload(recvMsg.Payload, 0) if err != nil { consensus.getLogger().Error().Err(err).Msg("ReadSignatureBitmapPayload failed!") return @@ -271,13 +282,11 @@ func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { return } curBlockNum := consensus.BlockNum() - consensus.mutex.Lock() - defer consensus.mutex.Unlock() for _, committedMsg := range consensus.FBFTLog.GetNotVerifiedCommittedMessages(blockObj.NumberU64(), blockObj.Header().ViewID().Uint64(), blockObj.Hash()) { if committedMsg != nil { consensus.onCommitted(committedMsg) } - if curBlockNum < consensus.getBlockNum() { + if curBlockNum < consensus.BlockNum() { consensus.getLogger().Info().Msg("[OnPrepared] Successfully caught up with committed message") break } @@ -286,6 +295,9 @@ func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + consensus.getLogger().Info(). Uint64("MsgBlockNum", recvMsg.BlockNum). Uint64("MsgViewID", recvMsg.ViewID). @@ -380,7 +392,7 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { return } - if consensus.isViewChangingMode() { + if consensus.IsViewChangingMode() { consensus.getLogger().Info().Msg("[OnCommitted] Still in ViewChanging mode, Exiting!!") return } @@ -394,6 +406,13 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { consensus.getLogger().Info().Msg("[OnCommitted] Start consensus timer (new block added)") consensus.consensusTimeout[timeoutConsensus].Start() } + + //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum) + if blk != nil { + //consensus.ReshardingNextLeader(blk) + } else { + //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum, "blk is nil") + } } // Collect private keys that are part of the current committee. @@ -401,7 +420,7 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { func (consensus *Consensus) getPriKeysInCommittee() []*bls.PrivateKeyWrapper { priKeys := []*bls.PrivateKeyWrapper{} for i, key := range consensus.priKey { - if !consensus.isValidatorInCommittee(key.Pub.Bytes) { + if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { continue } priKeys = append(priKeys, &consensus.priKey[i]) diff --git a/consensus/view_change.go b/consensus/view_change.go index aafdfd1210..2936fba018 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -1,7 +1,9 @@ package consensus import ( + "fmt" "math/big" + "sync" "time" "github.com/harmony-one/harmony/internal/chain" @@ -24,21 +26,26 @@ const MaxViewIDDiff = 249 // State contains current mode and current viewID type State struct { - mode Mode + mode Mode + modeMux sync.RWMutex // current view id in normal mode // it changes per successful consensus blockViewID uint64 + cViewMux sync.RWMutex // view changing id is used during view change mode // it is the next view id viewChangingID uint64 + viewMux sync.RWMutex isBackup bool } // Mode return the current node mode func (pm *State) Mode() Mode { + pm.modeMux.RLock() + defer pm.modeMux.RUnlock() return pm.mode } @@ -48,16 +55,22 @@ func (pm *State) SetMode(s Mode) { s = NormalBackup } + pm.modeMux.Lock() + defer pm.modeMux.Unlock() pm.mode = s } // GetCurBlockViewID return the current view id func (pm *State) GetCurBlockViewID() uint64 { + pm.cViewMux.RLock() + defer pm.cViewMux.RUnlock() return pm.blockViewID } // SetCurBlockViewID sets the current view id func (pm *State) SetCurBlockViewID(viewID uint64) uint64 { + pm.cViewMux.Lock() + defer pm.cViewMux.Unlock() pm.blockViewID = viewID return pm.blockViewID } @@ -65,18 +78,26 @@ func (pm *State) SetCurBlockViewID(viewID uint64) uint64 { // GetViewChangingID return the current view changing id // It is meaningful during view change mode func (pm *State) GetViewChangingID() uint64 { + pm.viewMux.RLock() + defer pm.viewMux.RUnlock() return pm.viewChangingID } // SetViewChangingID set the current view changing id // It is meaningful during view change mode func (pm *State) SetViewChangingID(id uint64) { + pm.viewMux.Lock() + defer pm.viewMux.Unlock() pm.viewChangingID = id } // GetViewChangeDuraion return the duration of the current view change // It increase in the power of difference betweeen view changing ID and current view ID func (pm *State) GetViewChangeDuraion() time.Duration { + pm.viewMux.RLock() + pm.cViewMux.RLock() + defer pm.viewMux.RUnlock() + defer pm.cViewMux.RUnlock() diff := int64(pm.viewChangingID - pm.blockViewID) return time.Duration(diff * diff * int64(viewChangeDuration)) } @@ -88,14 +109,14 @@ func (pm *State) SetIsBackup(isBackup bool) { // fallbackNextViewID return the next view ID and duration when there is an exception // to calculate the time-based viewId func (consensus *Consensus) fallbackNextViewID() (uint64, time.Duration) { - diff := int64(consensus.getViewChangingID() + 1 - consensus.getCurBlockViewID()) + diff := int64(consensus.GetViewChangingID() + 1 - consensus.GetCurBlockViewID()) if diff <= 0 { diff = int64(1) } consensus.getLogger().Error(). Int64("diff", diff). Msg("[fallbackNextViewID] use legacy viewID algorithm") - return consensus.getViewChangingID() + 1, time.Duration(diff * diff * int64(viewChangeDuration)) + return consensus.GetViewChangingID() + 1, time.Duration(diff * diff * int64(viewChangeDuration)) } // getNextViewID return the next view ID based on the timestamp @@ -141,6 +162,7 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { Uint64("stuckBlockViewID", stuckBlockViewID). Msg("[getNextViewID]") + fmt.Println("end getNextViewID: ", nextViewID, viewChangeDuration) // duration is always the fixed view change duration for synchronous view change return nextViewID, viewChangeDuration } @@ -152,9 +174,9 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrapper { gap := 1 - cur := consensus.getCurBlockViewID() + cur := consensus.GetCurBlockViewID() if viewID > cur { - gap = int(viewID - cur) + gap = int(viewID - consensus.GetCurBlockViewID()) } var lastLeaderPubKey *bls.PublicKeyWrapper var err error @@ -196,29 +218,24 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe Str("leaderPubKey", consensus.LeaderPubKey.Bytes.Hex()). Int("gap", gap). Uint64("newViewID", viewID). - Uint64("myCurBlockViewID", consensus.getCurBlockViewID()). + Uint64("myCurBlockViewID", consensus.GetCurBlockViewID()). Msg("[getNextLeaderKey] got leaderPubKey from coinbase") // wasFound, next := consensus.Decider.NthNext(lastLeaderPubKey, gap) // FIXME: rotate leader on harmony nodes only before fully externalization var wasFound bool var next *bls.PublicKeyWrapper - if blockchain != nil && blockchain.Config().IsLeaderRotation(epoch) { - if consensus.ShardID == shard.BeaconChainShardID { - wasFound, next = consensus.Decider.NthNextHmy( - shard.Schedule.InstanceForEpoch(epoch), - lastLeaderPubKey, - gap) - } else { - wasFound, next = consensus.Decider.NthNext( - lastLeaderPubKey, - gap) - } + if blockchain != nil && blockchain.Config().IsAllowlistEpoch(epoch) { + wasFound, next = consensus.Decider.NthNextHmyExt( + shard.Schedule.InstanceForEpoch(epoch), + lastLeaderPubKey, + gap) } else { wasFound, next = consensus.Decider.NthNextHmy( shard.Schedule.InstanceForEpoch(epoch), lastLeaderPubKey, gap) } + fmt.Println("wasfoundNext", consensus.Blockchain.Config().IsAllowlistEpoch(epoch), wasFound, next.Bytes.Hex(), lastLeaderPubKey.Bytes.Hex()) if !wasFound { consensus.getLogger().Warn(). Str("key", consensus.LeaderPubKey.Bytes.Hex()). @@ -240,26 +257,35 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - if consensus.disableViewChange || consensus.isBackup { + fmt.Printf("Message to send leader111: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) + if consensus.disableViewChange || consensus.IsBackup() { return } + consensus.mutex.Lock() + defer consensus.mutex.Unlock() consensus.consensusTimeout[timeoutConsensus].Stop() consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() - consensus.setViewChangingID(nextViewID) + //fmt.Println("startViewChange", nextViewID) + consensus.SetViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet // we use it this way as in many code we validate the messages // aganist the consensus.LeaderPubKey variable. // Ideally, we shall use another variable to keep track of the // leader pubkey in viewchange mode - consensus.LeaderPubKey = consensus.getNextLeaderKey(nextViewID) + consensus.pubKeyLock.Lock() + lpk := consensus.getNextLeaderKey(nextViewID) + consensus.LeaderPubKey = lpk + //fmt.Println("Message to send leader cur: ", consensus.LeaderPubKey.Bytes.Hex(), "next: ", lpk.Bytes.Hex()) + //fmt.Println("Message to send leader: ", consensus.LeaderPubKey.Bytes.Hex()) + consensus.pubKeyLock.Unlock() consensus.getLogger().Warn(). Uint64("nextViewID", nextViewID). - Uint64("viewChangingID", consensus.getViewChangingID()). + Uint64("viewChangingID", consensus.GetViewChangingID()). Dur("timeoutDuration", duration). Str("NextLeader", consensus.LeaderPubKey.Bytes.Hex()). Msg("[startViewChange]") @@ -276,7 +302,7 @@ func (consensus *Consensus) startViewChange() { if err := consensus.vc.InitPayload( consensus.FBFTLog, nextViewID, - consensus.getBlockNum(), + consensus.BlockNum(), consensus.priKey, members); err != nil { consensus.getLogger().Error().Err(err).Msg("[startViewChange] Init Payload Error") @@ -285,12 +311,14 @@ func (consensus *Consensus) startViewChange() { // for view change, send separate view change per public key // do not do multi-sign of view change message for _, key := range consensus.priKey { - if !consensus.isValidatorInCommittee(key.Pub.Bytes) { + if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { continue } + // Тут уже другой leader msgToSend := consensus.constructViewChangeMessage(&key) + fmt.Println("Message to send leader222: ", consensus.LeaderPubKey.Bytes.Hex()) if err := consensus.msgSender.SendWithRetry( - consensus.getBlockNum(), + consensus.BlockNum(), msg_pb.MessageType_VIEWCHANGE, []nodeconfig.GroupID{ nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, @@ -304,7 +332,7 @@ func (consensus *Consensus) startViewChange() { // startNewView stops the current view change func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.PrivateKeyWrapper, reset bool) error { - if !consensus.isViewChangingMode() { + if !consensus.IsViewChangingMode() { return errors.New("not in view changing mode anymore") } @@ -316,7 +344,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri } if err := consensus.msgSender.SendWithRetry( - consensus.getBlockNum(), + consensus.BlockNum(), msg_pb.MessageType_NEWVIEW, []nodeconfig.GroupID{ nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, @@ -333,8 +361,8 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri consensus.current.SetMode(Normal) consensus.consensusTimeout[timeoutViewChange].Stop() - consensus.setViewIDs(viewID) - consensus.resetViewChangeState() + consensus.SetViewIDs(viewID) + consensus.ResetViewChangeState() consensus.consensusTimeout[timeoutConsensus].Start() consensus.getLogger().Info(). @@ -344,15 +372,20 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri // TODO: consider make ResetState unified and only called in one place like finalizeCommit() if reset { - consensus.resetState() + consensus.ResetState() } - consensus.setLeaderPubKey(newLeaderPriKey.Pub) + fmt.Println("[startNewView]", newLeaderPriKey.Pub.Bytes.Hex()) + consensus.LeaderPubKey = newLeaderPriKey.Pub return nil } // onViewChange is called when the view change message is received. func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { + //fmt.Printf("[onViewChange] received view change message from %+v\n", recvMsg) + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + consensus.getLogger().Debug(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -361,7 +394,7 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { // if not leader, noop newLeaderKey := recvMsg.LeaderPubkey - newLeaderPriKey, err := consensus.getLeaderPrivateKey(newLeaderKey.Object) + newLeaderPriKey, err := consensus.GetLeaderPrivateKey(newLeaderKey.Object) if err != nil { consensus.getLogger().Debug(). Err(err). @@ -372,6 +405,13 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { return } + consensus.getLogger().Debug(). + Err(err). + Interface("SenderPubkeys", recvMsg.SenderPubkeys). + Str("NextLeader", recvMsg.LeaderPubkey.Bytes.Hex()). + Str("myBLSPubKey", consensus.priKey.GetPublicKeys().SerializeToHexStr()). + Msg("[onViewChange] I am the Leader") + if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) { consensus.getLogger().Info(). Int64("have", consensus.Decider.SignersCount(quorum.ViewChange)). @@ -414,7 +454,7 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { } // received enough view change messages, change state to normal consensus - if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) && consensus.isViewChangingMode() { + if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) && consensus.IsViewChangingMode() { // no previous prepared message, go straight to normal mode // and start proposing new block if consensus.vc.IsM1PayloadEmpty() { @@ -449,6 +489,11 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { // Or the validator will enter announce phase to wait for the new block proposed // from the new leader func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + + fmt.Printf("[onNewView] received new view message from %+v\n", recvMsg) + consensus.getLogger().Info(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -456,10 +501,10 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { Msg("[onNewView] Received NewView Message") // change view and leaderKey to keep in sync with network - if consensus.getBlockNum() != recvMsg.BlockNum { + if consensus.BlockNum() != recvMsg.BlockNum { consensus.getLogger().Warn(). Uint64("MsgBlockNum", recvMsg.BlockNum). - Uint64("myBlockNum", consensus.getBlockNum()). + Uint64("myBlockNum", consensus.BlockNum()). Msg("[onNewView] Invalid block number") return } @@ -493,7 +538,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { utils.CountOneBits(m3Mask.Bitmap) > utils.CountOneBits(m2Mask.Bitmap)) { // m1 is not empty, check it's valid blockHash := recvMsg.Payload[:32] - aggSig, mask, err := consensus.readSignatureBitmapPayload(recvMsg.Payload, 32, consensus.Decider.Participants()) + aggSig, mask, err := consensus.ReadSignatureBitmapPayload(recvMsg.Payload, 32) if err != nil { consensus.getLogger().Error().Err(err). Msg("[onNewView] ReadSignatureBitmapPayload Failed") @@ -527,7 +572,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { } } - if !consensus.isViewChangingMode() { + if !consensus.IsViewChangingMode() { consensus.getLogger().Info().Msg("Not in ViewChanging Mode.") return } @@ -535,9 +580,12 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.consensusTimeout[timeoutViewChange].Stop() // newView message verified success, override my state - consensus.setViewIDs(recvMsg.ViewID) + consensus.SetViewIDs(recvMsg.ViewID) + consensus.pubKeyLock.Lock() + fmt.Println("[onNewView1221] new leader key cur:", consensus.LeaderPubKey.Bytes.Hex(), " new: ", senderKey.Bytes.Hex()) consensus.LeaderPubKey = senderKey - consensus.resetViewChangeState() + consensus.pubKeyLock.Unlock() + consensus.ResetViewChangeState() consensus.msgSender.StopRetry(msg_pb.MessageType_VIEWCHANGE) @@ -546,7 +594,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.sendCommitMessages(preparedBlock) consensus.switchPhase("onNewView", FBFTCommit) } else { - consensus.resetState() + consensus.ResetState() consensus.getLogger().Info().Msg("onNewView === announce") } consensus.getLogger().Info(). @@ -558,13 +606,6 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // ResetViewChangeState resets the view change structure func (consensus *Consensus) ResetViewChangeState() { - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - consensus.resetViewChangeState() -} - -// ResetViewChangeState resets the view change structure -func (consensus *Consensus) resetViewChangeState() { consensus.getLogger().Info(). Str("Phase", consensus.phase.String()). Msg("[ResetViewChangeState] Resetting view change state") diff --git a/hmy/hmy.go b/hmy/hmy.go index 58e8d59edf..8089f04ba8 100644 --- a/hmy/hmy.go +++ b/hmy/hmy.go @@ -18,6 +18,7 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + "github.com/harmony-one/harmony/multibls" commonRPC "github.com/harmony-one/harmony/rpc/common" "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" @@ -96,6 +97,7 @@ type NodeAPI interface { GetStakingTransactionsCount(address, txType string) (uint64, error) GetTraceResultByHash(hash common.Hash) (json.RawMessage, error) IsCurrentlyLeader() bool + GetPublicKeys() multibls.PublicKeys IsOutOfSync(shardID uint32) bool SyncStatus(shardID uint32) (bool, uint64, uint64) SyncPeers() map[string]int diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index 10101d7673..1b71e981b5 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -197,6 +197,7 @@ func updateZeroLogLevel(level int) { zeroLogger = &childLogger } + // GetPort is useful for debugging, returns `--port` flag provided to executable. func GetPort() int { ok := false diff --git a/node/api.go b/node/api.go index ceda968084..debcb201f6 100644 --- a/node/api.go +++ b/node/api.go @@ -6,6 +6,7 @@ import ( "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy" "github.com/harmony-one/harmony/internal/tikv" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/rosetta" hmy_rpc "github.com/harmony-one/harmony/rpc" rpc_common "github.com/harmony-one/harmony/rpc/common" @@ -18,6 +19,11 @@ func (node *Node) IsCurrentlyLeader() bool { return node.Consensus.IsLeader() } +// GetPublicKeys exposes if node is currently the leader node +func (node *Node) GetPublicKeys() multibls.PublicKeys { + return node.Consensus.GetPrivateKeys().GetPublicKeys() +} + // PeerConnectivity .. func (node *Node) PeerConnectivity() (int, int, int) { return node.host.PeerConnectivity() diff --git a/node/node_newblock.go b/node/node_newblock.go index 5050e4d6a7..cbc4ad05ff 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -2,6 +2,7 @@ package node import ( "errors" + "fmt" "sort" "strings" "time" @@ -88,6 +89,8 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp newBlock, err := node.ProposeNewBlock(newCommitSigsChan) if err == nil { + fmt.Printf("ProposeNewBlock: #%d :%d @%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), newBlock.Header().ViewID().Int64(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) + if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). Msg("Block with the same number was already proposed, abort.") @@ -145,6 +148,7 @@ func (node *Node) ProposeNewBlock(commitSigs chan []byte) (*types.Block, error) if node.Blockchain().Config().IsStaking(header.Epoch()) { blsPubKeyBytes := leaderKey.Object.GetAddress() coinbase.SetBytes(blsPubKeyBytes[:]) + fmt.Println("coinbase.SetBytes leader: ", leaderKey.Bytes.Hex(), coinbase.Hex()) } emptyAddr := common.Address{} diff --git a/test/configs/local-resharding.txt b/test/configs/local-resharding.txt index 4ce91ece1c..fc3d3e97b4 100644 --- a/test/configs/local-resharding.txt +++ b/test/configs/local-resharding.txt @@ -1,25 +1,12 @@ 127.0.0.1 9000 validator .hmy/65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204.key -127.0.0.1 9002 validator .hmy/40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809.key -127.0.0.1 9004 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key -127.0.0.1 9006 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key -127.0.0.1 9008 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key -127.0.0.1 9010 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key -127.0.0.1 9012 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key -127.0.0.1 9014 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key -127.0.0.1 9016 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key -127.0.0.1 9018 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key -127.0.0.1 9020 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key -127.0.0.1 9022 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key -127.0.0.1 9200 explorer null 0 +127.0.0.1 9002 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key +127.0.0.1 9004 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key +127.0.0.1 9006 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key +127.0.0.1 9008 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key +127.0.0.1 9010 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key +127.0.0.1 9099 explorer null 0 127.0.0.1 9100 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key -127.0.0.1 9102 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key -127.0.0.1 9104 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key -127.0.0.1 9106 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key -127.0.0.1 9108 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key -127.0.0.1 9110 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key -127.0.0.1 9112 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key -127.0.0.1 9114 validator .hmy/f47238daef97d60deedbde5302d05dea5de67608f11f406576e363661f7dcbc4a1385948549b31a6c70f6fde8a391486.key -127.0.0.1 9116 validator .hmy/fc4b9c535ee91f015efff3f32fbb9d32cdd9bfc8a837bb3eee89b8fff653c7af2050a4e147ebe5c7233dc2d5df06ee0a.key -127.0.0.1 9118 validator .hmy/ca86e551ee42adaaa6477322d7db869d3e203c00d7b86c82ebee629ad79cb6d57b8f3db28336778ec2180e56a8e07296.key -127.0.0.1 9300 explorer null 1 \ No newline at end of file +127.0.0.1 9102 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key +127.0.0.1 9104 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key +127.0.0.1 9106 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key \ No newline at end of file diff --git a/test/deploy.sh b/test/deploy.sh index 9ab6943110..fe22de57f0 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -69,7 +69,7 @@ function launch_localnet() { if ${VERBOSE}; then verbosity=5 else - verbosity=3 + verbosity=5 fi base_args=(--log_folder "${log_folder}" --min_peers "${MIN}" --bootnodes "${BN_MA}" "--network_type=$NETWORK" --blspass file:"${ROOT}/.hmy/blspass.txt" "--dns=false" "--verbosity=${verbosity}" "--p2p.security.max-conn-per-ip=100") @@ -80,8 +80,11 @@ function launch_localnet() { while IFS='' read -r line || [[ -n "$line" ]]; do i=$((i + 1)) + + # Read config for i-th node form config file IFS=' ' read -r ip port mode bls_key shard node_config <<<"${line}" + echo "LINE: ${line} ${shard}" args=("${base_args[@]}" --ip "${ip}" --port "${port}" --key "/tmp/${ip}-${port}.key" --db_dir "${ROOT}/db-${ip}-${port}" "--broadcast_invalid_tx=false") if [[ -z "$ip" || -z "$port" ]]; then echo "skip empty node" From fc25a1a730f0453b8a9da3411f6dfb74ff358beb Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 1 Sep 2022 01:00:51 +0700 Subject: [PATCH 285/420] in progress. --- consensus/consensus_v2.go | 1 - 1 file changed, 1 deletion(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 16554d0393..16fa18d565 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -131,7 +131,6 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb } func (consensus *Consensus) finalCommit() { - // THIS IS NOT GOOD PLACE FOR LEADER SWITCHING numCommits := consensus.Decider.SignersCount(quorum.Commit) consensus.getLogger().Info(). From 686a80af9a3ac5842238e7dfaf83cee732ebaee9 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 1 Sep 2022 01:02:36 +0700 Subject: [PATCH 286/420] in progress. --- consensus/consensus_v2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 16fa18d565..9ab924c004 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -689,7 +689,7 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess // SetupForNewConsensus sets the state for new consensus func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) - curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) // first view id is going to be 2. + curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) prev := consensus.GetLeaderPubKey() idx := consensus.SetLeaderIndex(func(i int) int { if curBlockViewID%3 == 0 { From f92717283ca3446acc09d4c55c3cd997be71c7b3 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 5 Sep 2022 19:39:49 +0700 Subject: [PATCH 287/420] consensus check is forked --- api/service/explorer/service.go | 6 ++-- consensus/consensus.go | 2 -- consensus/consensus_service.go | 14 -------- consensus/consensus_v2.go | 63 +++++++++++++++++++++++---------- 4 files changed, 48 insertions(+), 37 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index d96c1bc56c..5887bb3a02 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -223,9 +223,9 @@ func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { for i := cur; i > 0; i-- { block := s.blockchain.GetBlockByNumber(i) - w.Write([]byte(fmt.Sprintf("%d ", i))) - w.Write([]byte(fmt.Sprintf("%s ", block.Header().ViewID().String()))) - w.Write([]byte(fmt.Sprintf("%s ", block.Header().Coinbase().Hash().Hex()))) + w.Write([]byte(fmt.Sprintf("#%d ", i))) + w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) + w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) } } diff --git a/consensus/consensus.go b/consensus/consensus.go index 69a4f4638f..ac15adc15f 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -71,8 +71,6 @@ type Consensus struct { priKey multibls.PrivateKeys // the publickey of leader LeaderPubKey *bls.PublicKeyWrapper - // index of leader in the list of validators. - LeaderIndex int // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e646504dde..d26ea8ced5 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -474,20 +474,6 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } -// SetLeaderIndex set the leader index. -func (consensus *Consensus) SetLeaderIndex(f func(int) int) (current int) { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() - consensus.LeaderIndex = f(consensus.LeaderIndex) - return consensus.LeaderIndex -} - -func (consensus *Consensus) GetLeaderIndex() int { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() - return consensus.LeaderIndex -} - // SetViewChangingID set the current view change ID func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 9ab924c004..a36d3d9212 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -5,6 +5,7 @@ import ( "context" "encoding/hex" "fmt" + "math/big" "sync/atomic" "time" @@ -691,27 +692,42 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) prev := consensus.GetLeaderPubKey() - idx := consensus.SetLeaderIndex(func(i int) int { - if curBlockViewID%3 == 0 { - return i + 1 + if consensus.Blockchain.Config().IsLeaderRotation(blk.Epoch()) { + epochBlockViewID, err := consensus.getEpochFirstBlockViewID(blk.Epoch()) + if err != nil { + consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", blk.Epoch().Uint64()) + return + } + if epochBlockViewID > curBlockViewID { + consensus.getLogger().Error().Msg("[SetupForNewConsensus] Epoch block viewID is greater than current block viewID") + return } - return i - }) - pps := consensus.Decider.Participants() - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = &pps[idx%len(pps)] - fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %q\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) - consensus.pubKeyLock.Unlock() - if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { - // leader changed - go func() { - fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - consensus.ReadySignal <- SyncProposal - - }() + diff := curBlockViewID - epochBlockViewID + + pps := consensus.Decider.Participants() + idx := (int(diff) / 3) % len(pps) + consensus.pubKeyLock.Lock() + fmt.Println("(int(diff)/3)%len(pps) == ", idx) + consensus.LeaderPubKey = &pps[idx] + fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + consensus.pubKeyLock.Unlock() + if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { + // leader changed + go func() { + fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + consensus.ReadySignal <- SyncProposal + + }() + } + } else { + fmt.Printf("SetupForNewConsensus0 :%d future v%d new: %s prev: %s %v\n", utils.GetPort(), curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] + consensus.pubKeyLock.Unlock() } + // Update consensus keys at last so the change of leader status doesn't mess up normal flow if blk.IsLastBlockInEpoch() { consensus.SetMode(consensus.UpdateConsensusInformation()) @@ -720,6 +736,17 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.ResetState() } +func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, error) { + if epoch.Uint64() == 0 { + return 0, nil + } + epochBlock := consensus.Blockchain.GetBlockByNumber(epoch.Uint64() - 1) + if epochBlock == nil { + return 0, errors.Errorf("block not found for number %d", epoch.Uint64()-1) + } + return epochBlock.Header().ViewID().Uint64() + 1, nil +} + func (consensus *Consensus) postCatchup(initBN uint64) { if initBN < consensus.BlockNum() { consensus.getLogger().Info(). From f3eb24c10d42c7b322365c5285c32944bdfd8485 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 5 Sep 2022 22:41:06 +0700 Subject: [PATCH 288/420] update master --- api/service/explorer/service.go | 2 ++ consensus/consensus_v2.go | 9 +++------ consensus/view_change.go | 2 +- node/node_newblock.go | 1 - 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 5887bb3a02..1ebfa94ba1 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -223,6 +223,8 @@ func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { for i := cur; i > 0; i-- { block := s.blockchain.GetBlockByNumber(i) + leaderPubKey, _ := chain.GetLeaderPubKeyFromCoinbase(s.backend.Blockchain(), block.Header()) + w.Write([]byte(fmt.Sprintf("%s ", leaderPubKey.Bytes.Hex()))) w.Write([]byte(fmt.Sprintf("#%d ", i))) w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index a36d3d9212..cdd641f906 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -708,21 +708,18 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg pps := consensus.Decider.Participants() idx := (int(diff) / 3) % len(pps) consensus.pubKeyLock.Lock() - fmt.Println("(int(diff)/3)%len(pps) == ", idx) + //fmt.Println("(int(diff)/3)%len(pps) == ", idx) consensus.LeaderPubKey = &pps[idx] - fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + //fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) consensus.pubKeyLock.Unlock() if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { // leader changed go func() { - fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) consensus.ReadySignal <- SyncProposal - }() } } else { - fmt.Printf("SetupForNewConsensus0 :%d future v%d new: %s prev: %s %v\n", utils.GetPort(), curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + fmt.Printf("SetupForNewConsensus0 Before LeaderRotation") consensus.pubKeyLock.Lock() consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] consensus.pubKeyLock.Unlock() diff --git a/consensus/view_change.go b/consensus/view_change.go index 2936fba018..9c045a9eef 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -257,7 +257,7 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - fmt.Printf("Message to send leader111: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) + fmt.Printf("[startViewChange]: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } diff --git a/node/node_newblock.go b/node/node_newblock.go index cbc4ad05ff..9e1870c933 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -148,7 +148,6 @@ func (node *Node) ProposeNewBlock(commitSigs chan []byte) (*types.Block, error) if node.Blockchain().Config().IsStaking(header.Epoch()) { blsPubKeyBytes := leaderKey.Object.GetAddress() coinbase.SetBytes(blsPubKeyBytes[:]) - fmt.Println("coinbase.SetBytes leader: ", leaderKey.Bytes.Hex(), coinbase.Hex()) } emptyAddr := common.Address{} From 83aa24846962f8885c2d113775fbc331e73d2805 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 6 Sep 2022 02:58:48 +0700 Subject: [PATCH 289/420] fix leader --- consensus/consensus.go | 1 + consensus/consensus_service.go | 45 +++++++++++++++++++----- consensus/consensus_v2.go | 64 ++++++++++++++++++++++++++++------ 3 files changed, 90 insertions(+), 20 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index ac15adc15f..7d95246f21 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -74,6 +74,7 @@ type Consensus struct { // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 + epoch uint64 // Blockhash - 32 byte blockHash [32]byte // Block to run consensus on diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index d26ea8ced5..d2747d19d9 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -77,6 +77,7 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) + consensus.pubKeyLock.Unlock() consensus.getLogger().Info().Msg("My Committee updated") for i := range pubKeys { consensus.getLogger().Info(). @@ -84,17 +85,24 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi Str("BLSPubKey", pubKeys[i].Bytes.Hex()). Msg("Member") } - - allKeys := consensus.Decider.Participants() - if len(allKeys) != 0 { - consensus.LeaderPubKey = &allKeys[0] - consensus.getLogger().Info(). - Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") + if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { + consensus.updateLeader() } else { - consensus.getLogger().Error(). - Msg("[UpdatePublicKeys] Participants is empty") + consensus.pubKeyLock.Lock() + allKeys := consensus.Decider.Participants() + consensus.pubKeyLock.Unlock() + if len(allKeys) != 0 { + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = &allKeys[0] + consensus.pubKeyLock.Unlock() + consensus.getLogger().Info(). + Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") + } else { + consensus.getLogger().Error(). + Msg("[UpdatePublicKeys] Participants is empty") + } } - consensus.pubKeyLock.Unlock() + // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. consensus.UpdateBitmaps() @@ -474,6 +482,15 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } +func (consensus *Consensus) SetCurEpoch(epoch uint64) { + fmt.Println("SetCurEpoch", epoch) + atomic.StoreUint64(&consensus.epoch, epoch) +} + +func (consensus *Consensus) GetCurEpoch() *big.Int { + return big.NewInt(int64(atomic.LoadUint64(&consensus.epoch))) +} + // SetViewChangingID set the current view change ID func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) @@ -608,3 +625,13 @@ func (consensus *Consensus) getLogger() *zerolog.Logger { Logger() return &logger } + +func UpdatePublicKeyDefault(consensus *Consensus) { + if allKeys := consensus.Decider.Participants(); len(allKeys) > 0 { + consensus.LeaderPubKey = &allKeys[0] + } +} + +func UpdatePublicKeyRotate(consensus *Consensus) { + //consensus +} diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index cdd641f906..1313f279be 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -11,7 +11,6 @@ import ( bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" - nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" @@ -687,15 +686,14 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess return nil } -// SetupForNewConsensus sets the state for new consensus -func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { - atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) - curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) +func (consensus *Consensus) updateLeader() { + curBlockViewID := consensus.GetCurBlockViewID() prev := consensus.GetLeaderPubKey() - if consensus.Blockchain.Config().IsLeaderRotation(blk.Epoch()) { - epochBlockViewID, err := consensus.getEpochFirstBlockViewID(blk.Epoch()) + epoch := consensus.GetCurEpoch() + if consensus.Blockchain.Config().IsLeaderRotation(epoch) { + epochBlockViewID, err := consensus.getEpochFirstBlockViewID(epoch) if err != nil { - consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", blk.Epoch().Uint64()) + consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", epoch) return } if epochBlockViewID > curBlockViewID { @@ -706,7 +704,8 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg diff := curBlockViewID - epochBlockViewID pps := consensus.Decider.Participants() - idx := (int(diff) / 3) % len(pps) + fmt.Println("diff ", diff, "epochBlockViewID: ", curBlockViewID, "epochBlockViewID", epochBlockViewID, (int(diff) / 5), len(pps), epoch) + idx := (int(diff) / 5) % len(pps) consensus.pubKeyLock.Lock() //fmt.Println("(int(diff)/3)%len(pps) == ", idx) consensus.LeaderPubKey = &pps[idx] @@ -718,8 +717,51 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.ReadySignal <- SyncProposal }() } + } +} + +// SetupForNewConsensus sets the state for new consensus +func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { + atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) + consensus.SetCurBlockViewID(committedMsg.ViewID + 1) + if blk.IsLastBlockInEpoch() { + consensus.SetCurEpoch(blk.Epoch().Uint64() + 1) + } else { + consensus.SetCurEpoch(blk.Epoch().Uint64()) + } + //prev := consensus.GetLeaderPubKey() + if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { + //consensus.updateLeader() + consensus.updateLeader() + + /*{ + epochBlockViewID, err := consensus.getEpochFirstBlockViewID(consensus.GetCurEpoch()) + if err != nil { + consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", blk.Epoch().Uint64()) + return + } + if epochBlockViewID > curBlockViewID { + consensus.getLogger().Error().Msg("[SetupForNewConsensus] Epoch block viewID is greater than current block viewID") + return + } + + diff := curBlockViewID - epochBlockViewID + + pps := consensus.Decider.Participants() + idx := (int(diff) / 3) % len(pps) + consensus.pubKeyLock.Lock() + //fmt.Println("(int(diff)/3)%len(pps) == ", idx) + consensus.LeaderPubKey = &pps[idx] + //fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + consensus.pubKeyLock.Unlock() + if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { + // leader changed + go func() { + consensus.ReadySignal <- SyncProposal + }() + } + }*/ } else { - fmt.Printf("SetupForNewConsensus0 Before LeaderRotation") consensus.pubKeyLock.Lock() consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] consensus.pubKeyLock.Unlock() @@ -737,7 +779,7 @@ func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, er if epoch.Uint64() == 0 { return 0, nil } - epochBlock := consensus.Blockchain.GetBlockByNumber(epoch.Uint64() - 1) + epochBlock := consensus.Blockchain.GetBlockByNumber(shard.Schedule.EpochLastBlock(epoch.Uint64() - 1)) if epochBlock == nil { return 0, errors.Errorf("block not found for number %d", epoch.Uint64()-1) } From 527bfa667a64cc13d0fab4f40c9035e79178457d Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 7 Sep 2022 22:20:05 +0700 Subject: [PATCH 290/420] check leader for N blocks --- consensus/consensus.go | 5 +++ consensus/consensus_service.go | 29 +++++++-------- consensus/consensus_v2.go | 66 ++++++++++++++++++++-------------- consensus/view_change.go | 2 +- 4 files changed, 59 insertions(+), 43 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 7d95246f21..f8910ceefd 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -187,6 +187,11 @@ func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper { func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } +func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = pub + consensus.pubKeyLock.Unlock() +} func (consensus *Consensus) setLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { consensus.LeaderPubKey = pub diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index d2747d19d9..0d7a6c5543 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -77,8 +77,20 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) + allKeys := consensus.Decider.Participants() consensus.pubKeyLock.Unlock() - consensus.getLogger().Info().Msg("My Committee updated") + if len(allKeys) != 0 { + first := consensus.Decider.FirstParticipant( + shard.Schedule.InstanceForEpoch(consensus.Blockchain.CurrentHeader().Epoch())) + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = first + consensus.pubKeyLock.Unlock() + consensus.getLogger().Info(). + Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") + } else { + consensus.getLogger().Error(). + Msg("[UpdatePublicKeys] Participants is empty") + } for i := range pubKeys { consensus.getLogger().Info(). Int("index", i). @@ -87,20 +99,6 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi } if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { consensus.updateLeader() - } else { - consensus.pubKeyLock.Lock() - allKeys := consensus.Decider.Participants() - consensus.pubKeyLock.Unlock() - if len(allKeys) != 0 { - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = &allKeys[0] - consensus.pubKeyLock.Unlock() - consensus.getLogger().Info(). - Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") - } else { - consensus.getLogger().Error(). - Msg("[UpdatePublicKeys] Participants is empty") - } } // reset states after update public keys @@ -483,7 +481,6 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { } func (consensus *Consensus) SetCurEpoch(epoch uint64) { - fmt.Println("SetCurEpoch", epoch) atomic.StoreUint64(&consensus.epoch, epoch) } diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 1313f279be..d3e8dc8eb7 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -4,13 +4,13 @@ import ( "bytes" "context" "encoding/hex" - "fmt" "math/big" "sync/atomic" "time" bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" + "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" @@ -393,7 +393,6 @@ func (consensus *Consensus) Start( consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() case newBlock := <-blockChannel: - //consensus.ReshardingNextLeader(newBlock) consensus.getLogger().Info(). Uint64("MsgBlockNum", newBlock.NumberU64()). Msg("[ConsensusMainLoop] Received Proposed New Block!") @@ -405,7 +404,6 @@ func (consensus *Consensus) Start( } // Sleep to wait for the full block time consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") - <-time.After(time.Until(consensus.NextBlockDue)) consensus.StartFinalityCount() @@ -530,7 +528,6 @@ func (consensus *Consensus) getLastMileBlocksAndMsg(bnStart uint64) ([]*types.Bl // preCommitAndPropose commit the current block with 67% commit signatures and start // proposing new block which will wait on the full commit signatures to finish func (consensus *Consensus) preCommitAndPropose(blk *types.Block) error { - //fmt.Println("preCommitAndPropose", utils.GetPort(), blk.NumberU64()) if blk == nil { return errors.New("block to pre-commit is nil") } @@ -687,30 +684,48 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess } func (consensus *Consensus) updateLeader() { - curBlockViewID := consensus.GetCurBlockViewID() prev := consensus.GetLeaderPubKey() epoch := consensus.GetCurEpoch() + curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() if consensus.Blockchain.Config().IsLeaderRotation(epoch) { - epochBlockViewID, err := consensus.getEpochFirstBlockViewID(epoch) - if err != nil { - consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", epoch) - return + leader := consensus.GetLeaderPubKey() + for i := uint64(0); i < 5; i++ { + header := consensus.Blockchain.GetHeaderByNumber(curNumber - i) + if header == nil { + return + } + // Previous epoch, we should not change leader. + if header.Epoch().Uint64() != epoch.Uint64() { + return + } + // Check if the same leader. + pub, err := chain.GetLeaderPubKeyFromCoinbase(consensus.Blockchain, header) + if err != nil { + utils.Logger().Error().Err(err).Msg("Failed to get leader public key from coinbase") + return + } + if !pub.Object.IsEqual(leader.Object) { + // Another leader. + return + } + } + // Passed all checks, we can change leader. + var ( + wasFound bool + next *bls.PublicKeyWrapper + ) + // The same leader for N blocks. + if consensus.Blockchain.Config().IsAllowlistEpoch(epoch) { + wasFound, next = consensus.Decider.NthNextHmyExt(shard.Schedule.InstanceForEpoch(epoch), leader, 1) + } else { + wasFound, next = consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) } - if epochBlockViewID > curBlockViewID { - consensus.getLogger().Error().Msg("[SetupForNewConsensus] Epoch block viewID is greater than current block viewID") + if !wasFound { + utils.Logger().Error().Msg("Failed to get next leader") return + } else { + consensus.SetLeaderPubKey(next) } - - diff := curBlockViewID - epochBlockViewID - - pps := consensus.Decider.Participants() - fmt.Println("diff ", diff, "epochBlockViewID: ", curBlockViewID, "epochBlockViewID", epochBlockViewID, (int(diff) / 5), len(pps), epoch) - idx := (int(diff) / 5) % len(pps) - consensus.pubKeyLock.Lock() - //fmt.Println("(int(diff)/3)%len(pps) == ", idx) - consensus.LeaderPubKey = &pps[idx] - //fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) - consensus.pubKeyLock.Unlock() if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { // leader changed go func() { @@ -729,6 +744,9 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg } else { consensus.SetCurEpoch(blk.Epoch().Uint64()) } + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] + consensus.pubKeyLock.Unlock() //prev := consensus.GetLeaderPubKey() if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { //consensus.updateLeader() @@ -761,10 +779,6 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg }() } }*/ - } else { - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] - consensus.pubKeyLock.Unlock() } // Update consensus keys at last so the change of leader status doesn't mess up normal flow diff --git a/consensus/view_change.go b/consensus/view_change.go index 9c045a9eef..55d3d56a8b 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -176,7 +176,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe cur := consensus.GetCurBlockViewID() if viewID > cur { - gap = int(viewID - consensus.GetCurBlockViewID()) + gap = int(viewID - cur) } var lastLeaderPubKey *bls.PublicKeyWrapper var err error From 1baa64c6d0fe40a4754d5de872b672e3105c5e71 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sat, 10 Sep 2022 18:02:45 +0700 Subject: [PATCH 291/420] fix --- api/service/explorer/service.go | 2 -- cmd/harmony/main.go | 3 --- consensus/consensus_service.go | 21 --------------------- consensus/quorum/quorum.go | 7 +------ 4 files changed, 1 insertion(+), 32 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 1ebfa94ba1..b13f73db61 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -115,8 +115,6 @@ func (s *Service) Run() *http.Server { s.router = mux.NewRouter() - fmt.Println("++", addr) - // Set up router for addresses. // Fetch addresses request, accepts parameter size: how much addresses to read, // parameter prefix: from which address prefix start diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 0527a0b8c3..f62b81acd8 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -266,8 +266,6 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { var err error - fmt.Println("OS: ", os.Args) - nodeconfigSetShardSchedule(hc) nodeconfig.SetShardingSchedule(shard.Schedule) nodeconfig.SetVersion(getHarmonyVersion()) @@ -787,7 +785,6 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // Set the consensus ID to be the current block number viewID := currentNode.Blockchain().CurrentBlock().Header().ViewID().Uint64() - fmt.Println("viewID:", viewID) currentConsensus.SetViewIDs(viewID + 1) utils.Logger().Info(). Uint64("viewID", viewID). diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 0d7a6c5543..96056b5231 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -219,7 +219,6 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } - fmt.Println("[checkViewID] Set LEADEER PUB KEY ", msg.SenderPubkeys[0].Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() @@ -403,7 +402,6 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { Str("leaderPubKey", leaderPubKey.Bytes.Hex()). Msg("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain") consensus.pubKeyLock.Lock() - fmt.Println("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain", leaderPubKey.Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = leaderPubKey consensus.pubKeyLock.Unlock() } @@ -498,15 +496,6 @@ func (consensus *Consensus) StartFinalityCount() { consensus.finalityCounter.Store(time.Now().UnixNano()) } -//func (consensus *Consensus) ReshardingNextLeader(newblock *types.Block) { -// consensus.pubKeyLock.Lock() -// fmt.Println("nextBlock1 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) -// consensus.LeaderPubKey = consensus.getNextLeaderKey(consensus.GetCurBlockViewID() + 1) -// fmt.Println("nextBlock2 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) -// consensus.pubKeyLock.Unlock() -// -//} - // FinishFinalityCount calculate the current finality func (consensus *Consensus) FinishFinalityCount() { d := time.Now().UnixNano() @@ -622,13 +611,3 @@ func (consensus *Consensus) getLogger() *zerolog.Logger { Logger() return &logger } - -func UpdatePublicKeyDefault(consensus *Consensus) { - if allKeys := consensus.Decider.Participants(); len(allKeys) > 0 { - consensus.LeaderPubKey = &allKeys[0] - } -} - -func UpdatePublicKeyRotate(consensus *Consensus) { - //consensus -} diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 52b765a88c..aba62fa53f 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -230,17 +230,12 @@ func (s *cIdentities) NthNextHmy(instance shardingconfig.Instance, pubKey *bls.P Msg("[NthNextHmy] pubKey not found") } numNodes := instance.NumHarmonyOperatedNodesPerShard() - //fmt.Println("??idx:", idx, numNodes) // sanity check to avoid out of bound access if numNodes <= 0 || numNodes > len(s.publicKeys) { numNodes = len(s.publicKeys) } idx = (idx + next) % numNodes - //fmt.Println("-------idx:", idx) - - new := &s.publicKeys[idx] - fmt.Println("NthNextHmy: ", pubKey.Bytes.Hex(), new.Bytes.Hex()) - return found, new + return found, &s.publicKeys[idx] } // NthNextHmyExt return the Nth next pubkey of Harmony + allowlist nodes, next can be negative number From 47706faf291e50b71d415fce3bbccda0350c83c2 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 14:04:09 +0700 Subject: [PATCH 292/420] fix --- consensus/consensus_service.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 96056b5231..25594e044e 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync/atomic" "time" @@ -468,7 +467,6 @@ func (consensus *Consensus) isLeader() bool { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { - fmt.Println("SetViewIDs", height) consensus.SetCurBlockViewID(height) consensus.SetViewChangingID(height) } From ebf0cd6ba8cfcb2c3fe1152041f6de5818b8246b Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 15:27:56 +0700 Subject: [PATCH 293/420] Cleanup and fix update pub keys. --- consensus/consensus_service.go | 31 +++++++++++++++--------------- consensus/consensus_v2.go | 35 ++-------------------------------- consensus/leader.go | 11 +++-------- consensus/quorum/quorum.go | 1 - consensus/view_change.go | 15 +-------------- 5 files changed, 21 insertions(+), 72 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 25594e044e..77996258e4 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync/atomic" "time" @@ -73,33 +74,31 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { + if utils.GetPort() == 9000 { + //utils.Logger().Info().Msg("UpdatePublicKeys") + fmt.Println("UpdatePublicKeys", len(pubKeys), len(allowlist)) + } // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) + consensus.getLogger().Info().Msg("My Committee updated") + for i := range pubKeys { + consensus.getLogger().Info(). + Int("index", i). + Str("BLSPubKey", pubKeys[i].Bytes.Hex()). + Msg("Member") + } + allKeys := consensus.Decider.Participants() - consensus.pubKeyLock.Unlock() if len(allKeys) != 0 { - first := consensus.Decider.FirstParticipant( - shard.Schedule.InstanceForEpoch(consensus.Blockchain.CurrentHeader().Epoch())) - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = first - consensus.pubKeyLock.Unlock() + consensus.LeaderPubKey = &allKeys[0] consensus.getLogger().Info(). Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") } else { consensus.getLogger().Error(). Msg("[UpdatePublicKeys] Participants is empty") } - for i := range pubKeys { - consensus.getLogger().Info(). - Int("index", i). - Str("BLSPubKey", pubKeys[i].Bytes.Hex()). - Msg("Member") - } - if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { - consensus.updateLeader() - } - + consensus.pubKeyLock.Unlock() // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. consensus.UpdateBitmaps() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index d3e8dc8eb7..fe685ca857 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -645,7 +645,6 @@ func (consensus *Consensus) tryCatchup() error { consensus.getLogger().Error().Err(err).Msg("[TryCatchup] Failed to add block to chain") return err } - //fmt.Println("tryCatchup ", utils.GetPort(), blk.NumberU64()) select { // TODO: Remove this when removing dns sync and stream sync is fully up case consensus.VerifiedNewBlock <- blk: @@ -683,7 +682,7 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess return nil } -func (consensus *Consensus) updateLeader() { +func (consensus *Consensus) rotateLeader() { prev := consensus.GetLeaderPubKey() epoch := consensus.GetCurEpoch() curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() @@ -747,38 +746,8 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.pubKeyLock.Lock() consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] consensus.pubKeyLock.Unlock() - //prev := consensus.GetLeaderPubKey() if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { - //consensus.updateLeader() - consensus.updateLeader() - - /*{ - epochBlockViewID, err := consensus.getEpochFirstBlockViewID(consensus.GetCurEpoch()) - if err != nil { - consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", blk.Epoch().Uint64()) - return - } - if epochBlockViewID > curBlockViewID { - consensus.getLogger().Error().Msg("[SetupForNewConsensus] Epoch block viewID is greater than current block viewID") - return - } - - diff := curBlockViewID - epochBlockViewID - - pps := consensus.Decider.Participants() - idx := (int(diff) / 3) % len(pps) - consensus.pubKeyLock.Lock() - //fmt.Println("(int(diff)/3)%len(pps) == ", idx) - consensus.LeaderPubKey = &pps[idx] - //fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) - consensus.pubKeyLock.Unlock() - if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { - // leader changed - go func() { - consensus.ReadySignal <- SyncProposal - }() - } - }*/ + consensus.rotateLeader() } // Update consensus keys at last so the change of leader status doesn't mess up normal flow diff --git a/consensus/leader.go b/consensus/leader.go index 21f3b770d5..0d9a65094d 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -191,16 +191,12 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { // TODO HERE - //if recvMsg.ViewID == 10 { - // return - //} + if recvMsg.ViewID == 21 { + return + } consensus.mutex.Lock() defer consensus.mutex.Unlock() //// Read - Start - if consensus.ShardID == 0 { - //fmt.Println("onCommit ", recvMsg.BlockNum) - } - if !consensus.isRightBlockNumAndViewID(recvMsg) { return } @@ -332,5 +328,4 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED) } - //fmt.Println("onCommit99: ", utils.GetPort(), recvMsg.BlockNum) } diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index aba62fa53f..da1551cdf9 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -75,7 +75,6 @@ type ParticipantTracker interface { Participants() multibls.PublicKeys IndexOf(bls.SerializedPublicKey) int ParticipantsCount() int64 - NthNext(*bls.PublicKeyWrapper, int) (bool, *bls.PublicKeyWrapper) NthNextHmy(shardingconfig.Instance, *bls.PublicKeyWrapper, int) (bool, *bls.PublicKeyWrapper) NthNextHmyExt(shardingconfig.Instance, *bls.PublicKeyWrapper, int) (bool, *bls.PublicKeyWrapper) FirstParticipant(shardingconfig.Instance) *bls.PublicKeyWrapper diff --git a/consensus/view_change.go b/consensus/view_change.go index 55d3d56a8b..3bd0c181f2 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync" "time" @@ -162,7 +161,6 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { Uint64("stuckBlockViewID", stuckBlockViewID). Msg("[getNextViewID]") - fmt.Println("end getNextViewID: ", nextViewID, viewChangeDuration) // duration is always the fixed view change duration for synchronous view change return nextViewID, viewChangeDuration } @@ -235,7 +233,6 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe lastLeaderPubKey, gap) } - fmt.Println("wasfoundNext", consensus.Blockchain.Config().IsAllowlistEpoch(epoch), wasFound, next.Bytes.Hex(), lastLeaderPubKey.Bytes.Hex()) if !wasFound { consensus.getLogger().Warn(). Str("key", consensus.LeaderPubKey.Bytes.Hex()). @@ -257,7 +254,6 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - fmt.Printf("[startViewChange]: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } @@ -268,7 +264,6 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() - //fmt.Println("startViewChange", nextViewID) consensus.SetViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet @@ -279,8 +274,6 @@ func (consensus *Consensus) startViewChange() { consensus.pubKeyLock.Lock() lpk := consensus.getNextLeaderKey(nextViewID) consensus.LeaderPubKey = lpk - //fmt.Println("Message to send leader cur: ", consensus.LeaderPubKey.Bytes.Hex(), "next: ", lpk.Bytes.Hex()) - //fmt.Println("Message to send leader: ", consensus.LeaderPubKey.Bytes.Hex()) consensus.pubKeyLock.Unlock() consensus.getLogger().Warn(). @@ -314,9 +307,7 @@ func (consensus *Consensus) startViewChange() { if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { continue } - // Тут уже другой leader msgToSend := consensus.constructViewChangeMessage(&key) - fmt.Println("Message to send leader222: ", consensus.LeaderPubKey.Bytes.Hex()) if err := consensus.msgSender.SendWithRetry( consensus.BlockNum(), msg_pb.MessageType_VIEWCHANGE, @@ -374,8 +365,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.ResetState() } - fmt.Println("[startNewView]", newLeaderPriKey.Pub.Bytes.Hex()) - consensus.LeaderPubKey = newLeaderPriKey.Pub + consensus.SetLeaderPubKey(newLeaderPriKey.Pub) return nil } @@ -492,8 +482,6 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.mutex.Lock() defer consensus.mutex.Unlock() - fmt.Printf("[onNewView] received new view message from %+v\n", recvMsg) - consensus.getLogger().Info(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -582,7 +570,6 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // newView message verified success, override my state consensus.SetViewIDs(recvMsg.ViewID) consensus.pubKeyLock.Lock() - fmt.Println("[onNewView1221] new leader key cur:", consensus.LeaderPubKey.Bytes.Hex(), " new: ", senderKey.Bytes.Hex()) consensus.LeaderPubKey = senderKey consensus.pubKeyLock.Unlock() consensus.ResetViewChangeState() From 9fc4611cc9473eb5fc213f9c6c55cc87dc00466f Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 16:32:36 +0700 Subject: [PATCH 294/420] Rotate leader. --- consensus/consensus.go | 1 - consensus/consensus_service.go | 8 -------- consensus/consensus_v2.go | 12 +++--------- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index f8910ceefd..d28fae9ed3 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -74,7 +74,6 @@ type Consensus struct { // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 - epoch uint64 // Blockhash - 32 byte blockHash [32]byte // Block to run consensus on diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 77996258e4..af344c6820 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -475,14 +475,6 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } -func (consensus *Consensus) SetCurEpoch(epoch uint64) { - atomic.StoreUint64(&consensus.epoch, epoch) -} - -func (consensus *Consensus) GetCurEpoch() *big.Int { - return big.NewInt(int64(atomic.LoadUint64(&consensus.epoch))) -} - // SetViewChangingID set the current view change ID func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index fe685ca857..7c4290fad2 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -682,9 +682,8 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess return nil } -func (consensus *Consensus) rotateLeader() { +func (consensus *Consensus) rotateLeader(epoch *big.Int) { prev := consensus.GetLeaderPubKey() - epoch := consensus.GetCurEpoch() curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() if consensus.Blockchain.Config().IsLeaderRotation(epoch) { leader := consensus.GetLeaderPubKey() @@ -738,16 +737,11 @@ func (consensus *Consensus) rotateLeader() { func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) consensus.SetCurBlockViewID(committedMsg.ViewID + 1) - if blk.IsLastBlockInEpoch() { - consensus.SetCurEpoch(blk.Epoch().Uint64() + 1) - } else { - consensus.SetCurEpoch(blk.Epoch().Uint64()) - } consensus.pubKeyLock.Lock() consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] consensus.pubKeyLock.Unlock() - if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { - consensus.rotateLeader() + if consensus.Blockchain.Config().IsLeaderRotation(blk.Epoch()) { + consensus.rotateLeader(blk.Epoch()) } // Update consensus keys at last so the change of leader status doesn't mess up normal flow From 261aa9bafd6c786718bf0e8d3810d4c49cdc8e80 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 13 Sep 2022 01:10:13 +0700 Subject: [PATCH 295/420] fix fix fix fix fix --- consensus/consensus_service.go | 5 ----- consensus/consensus_v2.go | 16 ++++++++++++---- consensus/leader.go | 4 ---- node/node_newblock.go | 2 +- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index af344c6820..e6be606f2e 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync/atomic" "time" @@ -74,10 +73,6 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { - if utils.GetPort() == 9000 { - //utils.Logger().Info().Msg("UpdatePublicKeys") - fmt.Println("UpdatePublicKeys", len(pubKeys), len(allowlist)) - } // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 7c4290fad2..3122a368b9 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -8,6 +8,7 @@ import ( "sync/atomic" "time" + "github.com/ethereum/go-ethereum/common" bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" "github.com/harmony-one/harmony/internal/chain" @@ -685,10 +686,11 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess func (consensus *Consensus) rotateLeader(epoch *big.Int) { prev := consensus.GetLeaderPubKey() curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() + utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), consensus.Blockchain.Config().IsLeaderRotation(epoch), consensus.Blockchain.Config().LeaderRotationBlocksCount) if consensus.Blockchain.Config().IsLeaderRotation(epoch) { leader := consensus.GetLeaderPubKey() - for i := uint64(0); i < 5; i++ { - header := consensus.Blockchain.GetHeaderByNumber(curNumber - i) + for i := 0; i < consensus.Blockchain.Config().LeaderRotationBlocksCount; i++ { + header := consensus.Blockchain.GetHeaderByNumber(curNumber - uint64(i)) if header == nil { return } @@ -740,8 +742,14 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.pubKeyLock.Lock() consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] consensus.pubKeyLock.Unlock() - if consensus.Blockchain.Config().IsLeaderRotation(blk.Epoch()) { - consensus.rotateLeader(blk.Epoch()) + var epoch *big.Int + if blk.IsLastBlockInEpoch() { + epoch = new(big.Int).Add(blk.Epoch(), common.Big1) + } else { + epoch = blk.Epoch() + } + if consensus.Blockchain.Config().IsLeaderRotation(epoch) { + consensus.rotateLeader(epoch) } // Update consensus keys at last so the change of leader status doesn't mess up normal flow diff --git a/consensus/leader.go b/consensus/leader.go index 0d9a65094d..8dbda2c589 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -190,10 +190,6 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { - // TODO HERE - if recvMsg.ViewID == 21 { - return - } consensus.mutex.Lock() defer consensus.mutex.Unlock() //// Read - Start diff --git a/node/node_newblock.go b/node/node_newblock.go index 9e1870c933..da6e302022 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -89,7 +89,7 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp newBlock, err := node.ProposeNewBlock(newCommitSigsChan) if err == nil { - fmt.Printf("ProposeNewBlock: #%d :%d @%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), newBlock.Header().ViewID().Int64(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) + fmt.Printf("ProposeNewBlock: #%d :%d e:%d, sh:%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), node.Blockchain().CurrentHeader().Epoch().Uint64(), node.Blockchain().ShardID(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). From 117cc2bfbf1b5ba7bec8c0d7abd565ca115a9a62 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 13 Sep 2022 02:15:25 +0700 Subject: [PATCH 296/420] Cleaned. --- api/service/explorer/service.go | 48 ------------------------------- cmd/harmony/main.go | 3 +- consensus/consensus_v2.go | 13 --------- consensus/validator.go | 14 --------- consensus/view_change.go | 8 ------ node/node_newblock.go | 3 -- test/configs/local-resharding.txt | 31 ++++++++++++++------ test/deploy.sh | 5 +--- 8 files changed, 24 insertions(+), 101 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index b13f73db61..b5ffa1b8b0 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -7,13 +7,11 @@ import ( "fmt" "net" "net/http" - "os" "path" "strconv" "time" "github.com/RoaringBitmap/roaring/roaring64" - "github.com/harmony-one/harmony/internal/common" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" ethCommon "github.com/ethereum/go-ethereum/common" @@ -121,9 +119,6 @@ func (s *Service) Run() *http.Server { s.router.Path("/addresses").Queries("size", "{[0-9]*?}", "prefix", "{[a-zA-Z0-9]*?}").HandlerFunc(s.GetAddresses).Methods("GET") s.router.Path("/addresses").HandlerFunc(s.GetAddresses) s.router.Path("/height").HandlerFunc(s.GetHeight) - s.router.Path("/leader").HandlerFunc(s.GetLeader) - s.router.Path("/blocks").HandlerFunc(s.GetBlocks) - s.router.Path("/halt").HandlerFunc(s.halt) // Set up router for supply info s.router.Path("/burn-addresses").Queries().HandlerFunc(s.GetInaccessibleAddressInfo).Methods("GET") @@ -191,49 +186,6 @@ type HeightResponse struct { S3 uint64 `json:"3,omitempty"` } -func (s *Service) GetLeader(w http.ResponseWriter, r *http.Request) { - if s.backend.IsCurrentlyLeader() { - w.Write([]byte("true ")) - } else { - w.Write([]byte("false")) - } - - keys := "" - for _, p := range s.backend.GetPublicKeys() { - addr := common.Address{} - addrBytes := p.Object.GetAddress() - addr.SetBytes(addrBytes[:]) - keys += fmt.Sprintf("%s ", addr.String()) - break - } - //blsPubKeyBytes := leaderKey.Object.GetAddress() - //coinbase.SetBytes(blsPubKeyBytes[:]) - - w.Write([]byte(fmt.Sprintf(" %d", s.blockchain.ShardID()))) - w.Write([]byte(fmt.Sprintf(" %s", s.Port))) - w.Write([]byte(fmt.Sprintf(" %s", keys))) - w.Write([]byte(fmt.Sprintf(" %s", s.backend.GetPublicKeys().SerializeToHexStr()))) - -} - -func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { - cur := s.blockchain.CurrentHeader().Number().Uint64() - - for i := cur; i > 0; i-- { - block := s.blockchain.GetBlockByNumber(i) - leaderPubKey, _ := chain.GetLeaderPubKeyFromCoinbase(s.backend.Blockchain(), block.Header()) - w.Write([]byte(fmt.Sprintf("%s ", leaderPubKey.Bytes.Hex()))) - w.Write([]byte(fmt.Sprintf("#%d ", i))) - w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) - w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) - w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) - } -} - -func (s *Service) halt(w http.ResponseWriter, r *http.Request) { - os.Exit(0) -} - // GetHeight returns heights of current and beacon chains if needed. func (s *Service) GetHeight(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index f62b81acd8..f01cb758ed 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -409,9 +409,8 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { if currentNode.NodeConfig.Role() == nodeconfig.Validator { currentNode.RegisterValidatorServices() } else if currentNode.NodeConfig.Role() == nodeconfig.ExplorerNode { - + currentNode.RegisterExplorerServices() } - currentNode.RegisterExplorerServices() currentNode.RegisterService(service.CrosslinkSending, crosslink_sending.New(currentNode, currentNode.Blockchain())) if hc.Pprof.Enabled { setupPprofService(currentNode, hc) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 3122a368b9..e87ea21012 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -660,8 +660,6 @@ func (consensus *Consensus) tryCatchup() error { } func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMessage) error { - // this function evaluates for all, leader and validators. - if consensus.Blockchain().CurrentBlock().NumberU64() < blk.NumberU64() { if _, err := consensus.Blockchain().InsertChain([]*types.Block{blk}, !consensus.FBFTLog.IsBlockVerified(blk.Hash())); err != nil { consensus.getLogger().Error().Err(err).Msg("[commitBlock] Failed to add block to chain") @@ -760,17 +758,6 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.ResetState() } -func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, error) { - if epoch.Uint64() == 0 { - return 0, nil - } - epochBlock := consensus.Blockchain.GetBlockByNumber(shard.Schedule.EpochLastBlock(epoch.Uint64() - 1)) - if epochBlock == nil { - return 0, errors.Errorf("block not found for number %d", epoch.Uint64()-1) - } - return epochBlock.Header().ViewID().Uint64() + 1, nil -} - func (consensus *Consensus) postCatchup(initBN uint64) { if initBN < consensus.BlockNum() { consensus.getLogger().Info(). diff --git a/consensus/validator.go b/consensus/validator.go index 013769395c..a73ac92eb3 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -18,7 +18,6 @@ import ( ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { - recvMsg, err := consensus.ParseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). @@ -27,9 +26,6 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Msg("[OnAnnounce] Unparseable leader message") return } - if consensus.ShardID == 0 { - //fmt.Println("onAnnounce called ", recvMsg.BlockNum) - } // NOTE let it handle its own logs if !consensus.onAnnounceSanityChecks(recvMsg) { @@ -188,9 +184,6 @@ func (consensus *Consensus) sendCommitMessages(blockObj *types.Block) { // if onPrepared accepts the prepared message from the leader, then // it will send a COMMIT message for the leader to receive on the network. func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { - if consensus.ShardID == 0 { - //fmt.Println("onPrepared", recvMsg.BlockNum) - } consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -406,13 +399,6 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { consensus.getLogger().Info().Msg("[OnCommitted] Start consensus timer (new block added)") consensus.consensusTimeout[timeoutConsensus].Start() } - - //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum) - if blk != nil { - //consensus.ReshardingNextLeader(blk) - } else { - //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum, "blk is nil") - } } // Collect private keys that are part of the current committee. diff --git a/consensus/view_change.go b/consensus/view_change.go index 3bd0c181f2..6631051dcc 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -372,7 +372,6 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri // onViewChange is called when the view change message is received. func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { - //fmt.Printf("[onViewChange] received view change message from %+v\n", recvMsg) consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -395,13 +394,6 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { return } - consensus.getLogger().Debug(). - Err(err). - Interface("SenderPubkeys", recvMsg.SenderPubkeys). - Str("NextLeader", recvMsg.LeaderPubkey.Bytes.Hex()). - Str("myBLSPubKey", consensus.priKey.GetPublicKeys().SerializeToHexStr()). - Msg("[onViewChange] I am the Leader") - if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) { consensus.getLogger().Info(). Int64("have", consensus.Decider.SignersCount(quorum.ViewChange)). diff --git a/node/node_newblock.go b/node/node_newblock.go index da6e302022..5050e4d6a7 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -2,7 +2,6 @@ package node import ( "errors" - "fmt" "sort" "strings" "time" @@ -89,8 +88,6 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp newBlock, err := node.ProposeNewBlock(newCommitSigsChan) if err == nil { - fmt.Printf("ProposeNewBlock: #%d :%d e:%d, sh:%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), node.Blockchain().CurrentHeader().Epoch().Uint64(), node.Blockchain().ShardID(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) - if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). Msg("Block with the same number was already proposed, abort.") diff --git a/test/configs/local-resharding.txt b/test/configs/local-resharding.txt index fc3d3e97b4..4ce91ece1c 100644 --- a/test/configs/local-resharding.txt +++ b/test/configs/local-resharding.txt @@ -1,12 +1,25 @@ 127.0.0.1 9000 validator .hmy/65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204.key -127.0.0.1 9002 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key -127.0.0.1 9004 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key -127.0.0.1 9006 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key -127.0.0.1 9008 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key -127.0.0.1 9010 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key -127.0.0.1 9099 explorer null 0 +127.0.0.1 9002 validator .hmy/40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809.key +127.0.0.1 9004 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key +127.0.0.1 9006 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key +127.0.0.1 9008 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key +127.0.0.1 9010 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key +127.0.0.1 9012 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key +127.0.0.1 9014 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key +127.0.0.1 9016 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key +127.0.0.1 9018 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key +127.0.0.1 9020 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key +127.0.0.1 9022 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key +127.0.0.1 9200 explorer null 0 127.0.0.1 9100 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key -127.0.0.1 9102 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key -127.0.0.1 9104 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key -127.0.0.1 9106 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key \ No newline at end of file +127.0.0.1 9102 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key +127.0.0.1 9104 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key +127.0.0.1 9106 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key +127.0.0.1 9108 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key +127.0.0.1 9110 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key +127.0.0.1 9112 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key +127.0.0.1 9114 validator .hmy/f47238daef97d60deedbde5302d05dea5de67608f11f406576e363661f7dcbc4a1385948549b31a6c70f6fde8a391486.key +127.0.0.1 9116 validator .hmy/fc4b9c535ee91f015efff3f32fbb9d32cdd9bfc8a837bb3eee89b8fff653c7af2050a4e147ebe5c7233dc2d5df06ee0a.key +127.0.0.1 9118 validator .hmy/ca86e551ee42adaaa6477322d7db869d3e203c00d7b86c82ebee629ad79cb6d57b8f3db28336778ec2180e56a8e07296.key +127.0.0.1 9300 explorer null 1 \ No newline at end of file diff --git a/test/deploy.sh b/test/deploy.sh index fe22de57f0..9ab6943110 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -69,7 +69,7 @@ function launch_localnet() { if ${VERBOSE}; then verbosity=5 else - verbosity=5 + verbosity=3 fi base_args=(--log_folder "${log_folder}" --min_peers "${MIN}" --bootnodes "${BN_MA}" "--network_type=$NETWORK" --blspass file:"${ROOT}/.hmy/blspass.txt" "--dns=false" "--verbosity=${verbosity}" "--p2p.security.max-conn-per-ip=100") @@ -80,11 +80,8 @@ function launch_localnet() { while IFS='' read -r line || [[ -n "$line" ]]; do i=$((i + 1)) - - # Read config for i-th node form config file IFS=' ' read -r ip port mode bls_key shard node_config <<<"${line}" - echo "LINE: ${line} ${shard}" args=("${base_args[@]}" --ip "${ip}" --port "${port}" --key "/tmp/${ip}-${port}.key" --db_dir "${ROOT}/db-${ip}-${port}" "--broadcast_invalid_tx=false") if [[ -z "$ip" || -z "$port" ]]; then echo "skip empty node" From f72199c9ba9f1e88ca20321dc7473c2585c3d953 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 19 Sep 2022 21:40:21 +0700 Subject: [PATCH 297/420] Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. --- consensus/consensus_v2.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index e87ea21012..5fe176c2f4 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -11,7 +11,6 @@ import ( "github.com/ethereum/go-ethereum/common" bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" - "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" @@ -697,7 +696,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { return } // Check if the same leader. - pub, err := chain.GetLeaderPubKeyFromCoinbase(consensus.Blockchain, header) + pub, err := consensus.Blockchain.GetLeaderPubKeyFromCoinbase(header) if err != nil { utils.Logger().Error().Err(err).Msg("Failed to get leader public key from coinbase") return @@ -708,16 +707,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { } } // Passed all checks, we can change leader. - var ( - wasFound bool - next *bls.PublicKeyWrapper - ) - // The same leader for N blocks. - if consensus.Blockchain.Config().IsAllowlistEpoch(epoch) { - wasFound, next = consensus.Decider.NthNextHmyExt(shard.Schedule.InstanceForEpoch(epoch), leader, 1) - } else { - wasFound, next = consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) - } + wasFound, next := consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) if !wasFound { utils.Logger().Error().Msg("Failed to get next leader") return From 808c9e67fe0a254f4c602afdd9089963fac663b1 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 9 Nov 2022 15:36:28 +0700 Subject: [PATCH 298/420] Fixed code review. --- consensus/consensus_v2.go | 2 +- hmy/hmy.go | 2 -- node/api.go | 6 ------ 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 5fe176c2f4..9d8702f63c 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -691,7 +691,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { if header == nil { return } - // Previous epoch, we should not change leader. + // Previous block was epoch block, we should not change leader. if header.Epoch().Uint64() != epoch.Uint64() { return } diff --git a/hmy/hmy.go b/hmy/hmy.go index 8089f04ba8..58e8d59edf 100644 --- a/hmy/hmy.go +++ b/hmy/hmy.go @@ -18,7 +18,6 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" - "github.com/harmony-one/harmony/multibls" commonRPC "github.com/harmony-one/harmony/rpc/common" "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" @@ -97,7 +96,6 @@ type NodeAPI interface { GetStakingTransactionsCount(address, txType string) (uint64, error) GetTraceResultByHash(hash common.Hash) (json.RawMessage, error) IsCurrentlyLeader() bool - GetPublicKeys() multibls.PublicKeys IsOutOfSync(shardID uint32) bool SyncStatus(shardID uint32) (bool, uint64, uint64) SyncPeers() map[string]int diff --git a/node/api.go b/node/api.go index debcb201f6..ceda968084 100644 --- a/node/api.go +++ b/node/api.go @@ -6,7 +6,6 @@ import ( "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy" "github.com/harmony-one/harmony/internal/tikv" - "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/rosetta" hmy_rpc "github.com/harmony-one/harmony/rpc" rpc_common "github.com/harmony-one/harmony/rpc/common" @@ -19,11 +18,6 @@ func (node *Node) IsCurrentlyLeader() bool { return node.Consensus.IsLeader() } -// GetPublicKeys exposes if node is currently the leader node -func (node *Node) GetPublicKeys() multibls.PublicKeys { - return node.Consensus.GetPrivateKeys().GetPublicKeys() -} - // PeerConnectivity .. func (node *Node) PeerConnectivity() (int, int, int) { return node.host.PeerConnectivity() From ce1e2594341699b1b10f6c7840916aaca4e49428 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 10 Nov 2022 12:57:57 +0700 Subject: [PATCH 299/420] Fix review comments. --- consensus/consensus.go | 5 +++ consensus/consensus_v2.go | 66 +++++++++++++++++++-------------------- 2 files changed, 38 insertions(+), 33 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index d28fae9ed3..a9fcc3cc52 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -186,6 +186,11 @@ func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper { func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } + +func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { + return consensus.LeaderPubKey +} + func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { consensus.pubKeyLock.Lock() consensus.LeaderPubKey = pub diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 9d8702f63c..315fc332e7 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -680,46 +680,46 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess return nil } +// rotateLeader rotates the leader to the next leader in the committee. +// This function must be called with enabled leader rotation. func (consensus *Consensus) rotateLeader(epoch *big.Int) { - prev := consensus.GetLeaderPubKey() + prev := consensus.getLeaderPubKey() curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), consensus.Blockchain.Config().IsLeaderRotation(epoch), consensus.Blockchain.Config().LeaderRotationBlocksCount) - if consensus.Blockchain.Config().IsLeaderRotation(epoch) { - leader := consensus.GetLeaderPubKey() - for i := 0; i < consensus.Blockchain.Config().LeaderRotationBlocksCount; i++ { - header := consensus.Blockchain.GetHeaderByNumber(curNumber - uint64(i)) - if header == nil { - return - } - // Previous block was epoch block, we should not change leader. - if header.Epoch().Uint64() != epoch.Uint64() { - return - } - // Check if the same leader. - pub, err := consensus.Blockchain.GetLeaderPubKeyFromCoinbase(header) - if err != nil { - utils.Logger().Error().Err(err).Msg("Failed to get leader public key from coinbase") - return - } - if !pub.Object.IsEqual(leader.Object) { - // Another leader. - return - } + leader := consensus.getLeaderPubKey() + for i := 0; i < consensus.Blockchain.Config().LeaderRotationBlocksCount; i++ { + header := consensus.Blockchain.GetHeaderByNumber(curNumber - uint64(i)) + if header == nil { + return } - // Passed all checks, we can change leader. - wasFound, next := consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) - if !wasFound { - utils.Logger().Error().Msg("Failed to get next leader") + // Previous epoch, we should not change leader. + if header.Epoch().Uint64() != epoch.Uint64() { return - } else { - consensus.SetLeaderPubKey(next) } - if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { - // leader changed - go func() { - consensus.ReadySignal <- SyncProposal - }() + // Check if the same leader. + pub, err := consensus.Blockchain.GetLeaderPubKeyFromCoinbase(header) + if err != nil { + utils.Logger().Error().Err(err).Msg("Failed to get leader public key from coinbase") + return } + if !pub.Object.IsEqual(leader.Object) { + // Another leader. + return + } + } + // Passed all checks, we can change leader. + wasFound, next := consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) + if !wasFound { + utils.Logger().Error().Msg("Failed to get next leader") + return + } else { + consensus.setLeaderPubKey(next) + } + if consensus.isLeader() && !consensus.getLeaderPubKey().Object.IsEqual(prev.Object) { + // leader changed + go func() { + consensus.ReadySignal <- SyncProposal + }() } } From 1c752ba7b0bad903ea7f9104d8781cbf1a754123 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 10 Nov 2022 17:00:38 +0700 Subject: [PATCH 300/420] Returned locks in rotateLeader. --- consensus/consensus_v2.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 315fc332e7..ae57632481 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -683,10 +683,10 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess // rotateLeader rotates the leader to the next leader in the committee. // This function must be called with enabled leader rotation. func (consensus *Consensus) rotateLeader(epoch *big.Int) { - prev := consensus.getLeaderPubKey() + prev := consensus.GetLeaderPubKey() curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), consensus.Blockchain.Config().IsLeaderRotation(epoch), consensus.Blockchain.Config().LeaderRotationBlocksCount) - leader := consensus.getLeaderPubKey() + leader := consensus.GetLeaderPubKey() for i := 0; i < consensus.Blockchain.Config().LeaderRotationBlocksCount; i++ { header := consensus.Blockchain.GetHeaderByNumber(curNumber - uint64(i)) if header == nil { @@ -713,9 +713,9 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { utils.Logger().Error().Msg("Failed to get next leader") return } else { - consensus.setLeaderPubKey(next) + consensus.SetLeaderPubKey(next) } - if consensus.isLeader() && !consensus.getLeaderPubKey().Object.IsEqual(prev.Object) { + if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { // leader changed go func() { consensus.ReadySignal <- SyncProposal From 63b00642273eee7a2b5f6909fd4f834fe0589941 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 17 Jan 2023 11:57:05 -0300 Subject: [PATCH 301/420] Rebased onto dev. --- consensus/consensus_v2.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index ae57632481..0b1ce28cf4 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -684,11 +684,12 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess // This function must be called with enabled leader rotation. func (consensus *Consensus) rotateLeader(epoch *big.Int) { prev := consensus.GetLeaderPubKey() - curNumber := consensus.Blockchain.CurrentHeader().Number().Uint64() - utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), consensus.Blockchain.Config().IsLeaderRotation(epoch), consensus.Blockchain.Config().LeaderRotationBlocksCount) + bc := consensus.Blockchain() + curNumber := bc.CurrentHeader().Number().Uint64() + utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), bc.Config().IsLeaderRotation(epoch), bc.Config().LeaderRotationBlocksCount) leader := consensus.GetLeaderPubKey() - for i := 0; i < consensus.Blockchain.Config().LeaderRotationBlocksCount; i++ { - header := consensus.Blockchain.GetHeaderByNumber(curNumber - uint64(i)) + for i := 0; i < bc.Config().LeaderRotationBlocksCount; i++ { + header := bc.GetHeaderByNumber(curNumber - uint64(i)) if header == nil { return } @@ -697,7 +698,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { return } // Check if the same leader. - pub, err := consensus.Blockchain.GetLeaderPubKeyFromCoinbase(header) + pub, err := bc.GetLeaderPubKeyFromCoinbase(header) if err != nil { utils.Logger().Error().Err(err).Msg("Failed to get leader public key from coinbase") return @@ -736,7 +737,7 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg } else { epoch = blk.Epoch() } - if consensus.Blockchain.Config().IsLeaderRotation(epoch) { + if consensus.Blockchain().Config().IsLeaderRotation(epoch) { consensus.rotateLeader(epoch) } From 13a85b5771f129ad4507ecf366f34ff6e8a68e8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <“Gheis.Mohammadi@gmail.com”> Date: Tue, 27 Dec 2022 16:49:02 +0800 Subject: [PATCH 302/420] staged stream sync v1.0 --- go.mod | 1 + hmy/downloader/metric.go | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 18e16a0c05..6bbfc03980 100644 --- a/go.mod +++ b/go.mod @@ -166,6 +166,7 @@ require ( github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect + github.com/libp2p/go-libp2p-core v0.20.1 // indirect github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect github.com/libp2p/go-msgio v0.2.0 // indirect diff --git a/hmy/downloader/metric.go b/hmy/downloader/metric.go index 2995db7b31..9338fd1304 100644 --- a/hmy/downloader/metric.go +++ b/hmy/downloader/metric.go @@ -23,7 +23,7 @@ var ( consensusTriggeredDownloadCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "consensus_trigger", Help: "number of times consensus triggered download task", }, @@ -33,7 +33,7 @@ var ( longRangeSyncedBlockCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "num_blocks_synced_long_range", Help: "number of blocks synced in long range sync", }, @@ -43,7 +43,7 @@ var ( longRangeFailInsertedBlockCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "num_blocks_failed_long_range", Help: "number of blocks failed to insert into change in long range sync", }, @@ -53,7 +53,7 @@ var ( numShortRangeCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "num_short_range", Help: "number of short range sync is triggered", }, @@ -63,7 +63,7 @@ var ( numFailedDownloadCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "failed_download", Help: "number of downloading is failed", }, @@ -73,7 +73,7 @@ var ( numBlocksInsertedShortRangeHistogramVec = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "num_blocks_inserted_short_range", Help: "number of blocks inserted for each short range sync", // Buckets: 0, 1, 2, 4, +INF (capped at 10) @@ -85,7 +85,7 @@ var ( numBlocksInsertedBeaconHelperCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "downloader", + Subsystem: "StreamSync", Name: "num_blocks_inserted_beacon_helper", Help: "number of blocks inserted from beacon helper", }, From da1838a75f8fbc24e004f7101744deddca78f35d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Thu, 29 Dec 2022 11:36:58 +0800 Subject: [PATCH 303/420] refactor errors, rename metrics --- hmy/downloader/metric.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/hmy/downloader/metric.go b/hmy/downloader/metric.go index 9338fd1304..2995db7b31 100644 --- a/hmy/downloader/metric.go +++ b/hmy/downloader/metric.go @@ -23,7 +23,7 @@ var ( consensusTriggeredDownloadCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "consensus_trigger", Help: "number of times consensus triggered download task", }, @@ -33,7 +33,7 @@ var ( longRangeSyncedBlockCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "num_blocks_synced_long_range", Help: "number of blocks synced in long range sync", }, @@ -43,7 +43,7 @@ var ( longRangeFailInsertedBlockCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "num_blocks_failed_long_range", Help: "number of blocks failed to insert into change in long range sync", }, @@ -53,7 +53,7 @@ var ( numShortRangeCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "num_short_range", Help: "number of short range sync is triggered", }, @@ -63,7 +63,7 @@ var ( numFailedDownloadCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "failed_download", Help: "number of downloading is failed", }, @@ -73,7 +73,7 @@ var ( numBlocksInsertedShortRangeHistogramVec = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "num_blocks_inserted_short_range", Help: "number of blocks inserted for each short range sync", // Buckets: 0, 1, 2, 4, +INF (capped at 10) @@ -85,7 +85,7 @@ var ( numBlocksInsertedBeaconHelperCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "hmy", - Subsystem: "StreamSync", + Subsystem: "downloader", Name: "num_blocks_inserted_beacon_helper", Help: "number of blocks inserted from beacon helper", }, From f799b09bea17d69650b8f98349c13afecdcd7dd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Thu, 12 Jan 2023 08:41:46 +0800 Subject: [PATCH 304/420] fix p2p discovery test issue --- p2p/discovery/option_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/discovery/option_test.go b/p2p/discovery/option_test.go index 1829e77c85..e782151f59 100644 --- a/p2p/discovery/option_test.go +++ b/p2p/discovery/option_test.go @@ -40,14 +40,14 @@ func TestDHTOption_getLibp2pRawOptions(t *testing.T) { opt: DHTConfig{ BootNodes: testAddrStr, }, - expLen: 1, + expLen: 2, }, { opt: DHTConfig{ BootNodes: testAddrStr, DataStoreFile: &validPath, }, - expLen: 2, + expLen: 3, }, { opt: DHTConfig{ From a65ab985471bcfeb719f5f6d9838a7b5538fdc0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Mon, 13 Feb 2023 15:24:35 +0800 Subject: [PATCH 305/420] add watermark low/high options for p2p connection manager --- p2p/discovery/option_test.go | 4 ++-- rosetta/infra/harmony-mainnet.conf | 2 ++ rosetta/infra/harmony-pstn.conf | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/p2p/discovery/option_test.go b/p2p/discovery/option_test.go index e782151f59..1829e77c85 100644 --- a/p2p/discovery/option_test.go +++ b/p2p/discovery/option_test.go @@ -40,14 +40,14 @@ func TestDHTOption_getLibp2pRawOptions(t *testing.T) { opt: DHTConfig{ BootNodes: testAddrStr, }, - expLen: 2, + expLen: 1, }, { opt: DHTConfig{ BootNodes: testAddrStr, DataStoreFile: &validPath, }, - expLen: 3, + expLen: 2, }, { opt: DHTConfig{ diff --git a/rosetta/infra/harmony-mainnet.conf b/rosetta/infra/harmony-mainnet.conf index 8d51609cb1..051324e19f 100644 --- a/rosetta/infra/harmony-mainnet.conf +++ b/rosetta/infra/harmony-mainnet.conf @@ -74,6 +74,8 @@ Version = "2.5.13" MaxConnsPerIP = 10 MaxPeers = 0 Port = 9000 + ConnManagerLowWatermark = 160 + ConnManagerHighWatermark = 192 WaitForEachPeerToConnect = false [Pprof] diff --git a/rosetta/infra/harmony-pstn.conf b/rosetta/infra/harmony-pstn.conf index 1bb865c1a5..bc7df55bc0 100644 --- a/rosetta/infra/harmony-pstn.conf +++ b/rosetta/infra/harmony-pstn.conf @@ -74,6 +74,8 @@ Version = "2.5.13" MaxConnsPerIP = 10 MaxPeers = 0 Port = 9000 + ConnManagerLowWatermark = 160 + ConnManagerHighWatermark = 192 WaitForEachPeerToConnect = false [Pprof] From b5769d71c0a04512bbb7121c6a8fbb7a97785db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Tue, 14 Feb 2023 08:09:08 +0800 Subject: [PATCH 306/420] fix dedent --- rosetta/infra/harmony-mainnet.conf | 2 +- rosetta/infra/harmony-pstn.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rosetta/infra/harmony-mainnet.conf b/rosetta/infra/harmony-mainnet.conf index 051324e19f..6460075607 100644 --- a/rosetta/infra/harmony-mainnet.conf +++ b/rosetta/infra/harmony-mainnet.conf @@ -75,7 +75,7 @@ Version = "2.5.13" MaxPeers = 0 Port = 9000 ConnManagerLowWatermark = 160 - ConnManagerHighWatermark = 192 + ConnManagerHighWatermark = 192 WaitForEachPeerToConnect = false [Pprof] diff --git a/rosetta/infra/harmony-pstn.conf b/rosetta/infra/harmony-pstn.conf index bc7df55bc0..da2ab27847 100644 --- a/rosetta/infra/harmony-pstn.conf +++ b/rosetta/infra/harmony-pstn.conf @@ -75,7 +75,7 @@ Version = "2.5.13" MaxPeers = 0 Port = 9000 ConnManagerLowWatermark = 160 - ConnManagerHighWatermark = 192 + ConnManagerHighWatermark = 192 WaitForEachPeerToConnect = false [Pprof] From 24d3045bd36ff81d2f434437945294918d007948 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 1 Sep 2022 00:53:16 +0700 Subject: [PATCH 307/420] in progress. --- api/service/explorer/service.go | 48 +++++++++++++++++++++++++++++++ cmd/harmony/main.go | 6 +++- consensus/consensus.go | 2 ++ consensus/consensus_service.go | 27 +++++++++++++++++ consensus/consensus_v2.go | 31 ++++++++++++++++++-- consensus/leader.go | 9 ++++++ consensus/quorum/quorum.go | 7 ++++- consensus/validator.go | 14 +++++++++ consensus/view_change.go | 19 ++++++++++++ hmy/hmy.go | 2 ++ node/api.go | 6 ++++ node/node_newblock.go | 4 +++ test/configs/local-resharding.txt | 31 ++++++-------------- test/deploy.sh | 5 +++- 14 files changed, 184 insertions(+), 27 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index b5ffa1b8b0..d96c1bc56c 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -7,11 +7,13 @@ import ( "fmt" "net" "net/http" + "os" "path" "strconv" "time" "github.com/RoaringBitmap/roaring/roaring64" + "github.com/harmony-one/harmony/internal/common" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" ethCommon "github.com/ethereum/go-ethereum/common" @@ -113,12 +115,17 @@ func (s *Service) Run() *http.Server { s.router = mux.NewRouter() + fmt.Println("++", addr) + // Set up router for addresses. // Fetch addresses request, accepts parameter size: how much addresses to read, // parameter prefix: from which address prefix start s.router.Path("/addresses").Queries("size", "{[0-9]*?}", "prefix", "{[a-zA-Z0-9]*?}").HandlerFunc(s.GetAddresses).Methods("GET") s.router.Path("/addresses").HandlerFunc(s.GetAddresses) s.router.Path("/height").HandlerFunc(s.GetHeight) + s.router.Path("/leader").HandlerFunc(s.GetLeader) + s.router.Path("/blocks").HandlerFunc(s.GetBlocks) + s.router.Path("/halt").HandlerFunc(s.halt) // Set up router for supply info s.router.Path("/burn-addresses").Queries().HandlerFunc(s.GetInaccessibleAddressInfo).Methods("GET") @@ -186,6 +193,47 @@ type HeightResponse struct { S3 uint64 `json:"3,omitempty"` } +func (s *Service) GetLeader(w http.ResponseWriter, r *http.Request) { + if s.backend.IsCurrentlyLeader() { + w.Write([]byte("true ")) + } else { + w.Write([]byte("false")) + } + + keys := "" + for _, p := range s.backend.GetPublicKeys() { + addr := common.Address{} + addrBytes := p.Object.GetAddress() + addr.SetBytes(addrBytes[:]) + keys += fmt.Sprintf("%s ", addr.String()) + break + } + //blsPubKeyBytes := leaderKey.Object.GetAddress() + //coinbase.SetBytes(blsPubKeyBytes[:]) + + w.Write([]byte(fmt.Sprintf(" %d", s.blockchain.ShardID()))) + w.Write([]byte(fmt.Sprintf(" %s", s.Port))) + w.Write([]byte(fmt.Sprintf(" %s", keys))) + w.Write([]byte(fmt.Sprintf(" %s", s.backend.GetPublicKeys().SerializeToHexStr()))) + +} + +func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { + cur := s.blockchain.CurrentHeader().Number().Uint64() + + for i := cur; i > 0; i-- { + block := s.blockchain.GetBlockByNumber(i) + w.Write([]byte(fmt.Sprintf("%d ", i))) + w.Write([]byte(fmt.Sprintf("%s ", block.Header().ViewID().String()))) + w.Write([]byte(fmt.Sprintf("%s ", block.Header().Coinbase().Hash().Hex()))) + w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) + } +} + +func (s *Service) halt(w http.ResponseWriter, r *http.Request) { + os.Exit(0) +} + // GetHeight returns heights of current and beacon chains if needed. func (s *Service) GetHeight(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index f01cb758ed..0527a0b8c3 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -266,6 +266,8 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { var err error + fmt.Println("OS: ", os.Args) + nodeconfigSetShardSchedule(hc) nodeconfig.SetShardingSchedule(shard.Schedule) nodeconfig.SetVersion(getHarmonyVersion()) @@ -409,8 +411,9 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { if currentNode.NodeConfig.Role() == nodeconfig.Validator { currentNode.RegisterValidatorServices() } else if currentNode.NodeConfig.Role() == nodeconfig.ExplorerNode { - currentNode.RegisterExplorerServices() + } + currentNode.RegisterExplorerServices() currentNode.RegisterService(service.CrosslinkSending, crosslink_sending.New(currentNode, currentNode.Blockchain())) if hc.Pprof.Enabled { setupPprofService(currentNode, hc) @@ -784,6 +787,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // Set the consensus ID to be the current block number viewID := currentNode.Blockchain().CurrentBlock().Header().ViewID().Uint64() + fmt.Println("viewID:", viewID) currentConsensus.SetViewIDs(viewID + 1) utils.Logger().Info(). Uint64("viewID", viewID). diff --git a/consensus/consensus.go b/consensus/consensus.go index a9fcc3cc52..8477eb2aa9 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -71,6 +71,8 @@ type Consensus struct { priKey multibls.PrivateKeys // the publickey of leader LeaderPubKey *bls.PublicKeyWrapper + // index of leader in the list of validators. + LeaderIndex int // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e6be606f2e..e646504dde 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync/atomic" "time" @@ -212,6 +213,7 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } + fmt.Println("[checkViewID] Set LEADEER PUB KEY ", msg.SenderPubkeys[0].Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() @@ -395,6 +397,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { Str("leaderPubKey", leaderPubKey.Bytes.Hex()). Msg("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain") consensus.pubKeyLock.Lock() + fmt.Println("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain", leaderPubKey.Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = leaderPubKey consensus.pubKeyLock.Unlock() } @@ -461,6 +464,7 @@ func (consensus *Consensus) isLeader() bool { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { + fmt.Println("SetViewIDs", height) consensus.SetCurBlockViewID(height) consensus.SetViewChangingID(height) } @@ -470,6 +474,20 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } +// SetLeaderIndex set the leader index. +func (consensus *Consensus) SetLeaderIndex(f func(int) int) (current int) { + consensus.pubKeyLock.Lock() + defer consensus.pubKeyLock.Unlock() + consensus.LeaderIndex = f(consensus.LeaderIndex) + return consensus.LeaderIndex +} + +func (consensus *Consensus) GetLeaderIndex() int { + consensus.pubKeyLock.Lock() + defer consensus.pubKeyLock.Unlock() + return consensus.LeaderIndex +} + // SetViewChangingID set the current view change ID func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) @@ -480,6 +498,15 @@ func (consensus *Consensus) StartFinalityCount() { consensus.finalityCounter.Store(time.Now().UnixNano()) } +//func (consensus *Consensus) ReshardingNextLeader(newblock *types.Block) { +// consensus.pubKeyLock.Lock() +// fmt.Println("nextBlock1 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) +// consensus.LeaderPubKey = consensus.getNextLeaderKey(consensus.GetCurBlockViewID() + 1) +// fmt.Println("nextBlock2 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) +// consensus.pubKeyLock.Unlock() +// +//} + // FinishFinalityCount calculate the current finality func (consensus *Consensus) FinishFinalityCount() { d := time.Now().UnixNano() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 0b1ce28cf4..bf042c62d3 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/hex" + "fmt" "math/big" "sync/atomic" "time" @@ -131,6 +132,7 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb } func (consensus *Consensus) finalCommit() { + // THIS IS NOT GOOD PLACE FOR LEADER SWITCHING numCommits := consensus.Decider.SignersCount(quorum.Commit) consensus.getLogger().Info(). @@ -393,6 +395,7 @@ func (consensus *Consensus) Start( consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() case newBlock := <-blockChannel: + //consensus.ReshardingNextLeader(newBlock) consensus.getLogger().Info(). Uint64("MsgBlockNum", newBlock.NumberU64()). Msg("[ConsensusMainLoop] Received Proposed New Block!") @@ -404,6 +407,7 @@ func (consensus *Consensus) Start( } // Sleep to wait for the full block time consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") + <-time.After(time.Until(consensus.NextBlockDue)) consensus.StartFinalityCount() @@ -528,6 +532,7 @@ func (consensus *Consensus) getLastMileBlocksAndMsg(bnStart uint64) ([]*types.Bl // preCommitAndPropose commit the current block with 67% commit signatures and start // proposing new block which will wait on the full commit signatures to finish func (consensus *Consensus) preCommitAndPropose(blk *types.Block) error { + //fmt.Println("preCommitAndPropose", utils.GetPort(), blk.NumberU64()) if blk == nil { return errors.New("block to pre-commit is nil") } @@ -645,6 +650,7 @@ func (consensus *Consensus) tryCatchup() error { consensus.getLogger().Error().Err(err).Msg("[TryCatchup] Failed to add block to chain") return err } + //fmt.Println("tryCatchup ", utils.GetPort(), blk.NumberU64()) select { // TODO: Remove this when removing dns sync and stream sync is fully up case consensus.VerifiedNewBlock <- blk: @@ -659,6 +665,8 @@ func (consensus *Consensus) tryCatchup() error { } func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMessage) error { + // this function evaluates for all, leader and validators. + if consensus.Blockchain().CurrentBlock().NumberU64() < blk.NumberU64() { if _, err := consensus.Blockchain().InsertChain([]*types.Block{blk}, !consensus.FBFTLog.IsBlockVerified(blk.Hash())); err != nil { consensus.getLogger().Error().Err(err).Msg("[commitBlock] Failed to add block to chain") @@ -727,10 +735,29 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { // SetupForNewConsensus sets the state for new consensus func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) - consensus.SetCurBlockViewID(committedMsg.ViewID + 1) + curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) // first view id is going to be 2. + prev := consensus.GetLeaderPubKey() + idx := consensus.SetLeaderIndex(func(i int) int { + if curBlockViewID%3 == 0 { + return i + 1 + } + return i + }) + pps := consensus.Decider.Participants() consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] + consensus.LeaderPubKey = &pps[idx%len(pps)] + fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %q\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) consensus.pubKeyLock.Unlock() + if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { + // leader changed + go func() { + fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + consensus.ReadySignal <- SyncProposal + + }() + + } var epoch *big.Int if blk.IsLastBlockInEpoch() { epoch = new(big.Int).Add(blk.Epoch(), common.Big1) diff --git a/consensus/leader.go b/consensus/leader.go index 8dbda2c589..21f3b770d5 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -190,9 +190,17 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { + // TODO HERE + //if recvMsg.ViewID == 10 { + // return + //} consensus.mutex.Lock() defer consensus.mutex.Unlock() //// Read - Start + if consensus.ShardID == 0 { + //fmt.Println("onCommit ", recvMsg.BlockNum) + } + if !consensus.isRightBlockNumAndViewID(recvMsg) { return } @@ -324,4 +332,5 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED) } + //fmt.Println("onCommit99: ", utils.GetPort(), recvMsg.BlockNum) } diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index da1551cdf9..6aa9b0dd79 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -229,12 +229,17 @@ func (s *cIdentities) NthNextHmy(instance shardingconfig.Instance, pubKey *bls.P Msg("[NthNextHmy] pubKey not found") } numNodes := instance.NumHarmonyOperatedNodesPerShard() + //fmt.Println("??idx:", idx, numNodes) // sanity check to avoid out of bound access if numNodes <= 0 || numNodes > len(s.publicKeys) { numNodes = len(s.publicKeys) } idx = (idx + next) % numNodes - return found, &s.publicKeys[idx] + //fmt.Println("-------idx:", idx) + + new := &s.publicKeys[idx] + fmt.Println("NthNextHmy: ", pubKey.Bytes.Hex(), new.Bytes.Hex()) + return found, new } // NthNextHmyExt return the Nth next pubkey of Harmony + allowlist nodes, next can be negative number diff --git a/consensus/validator.go b/consensus/validator.go index a73ac92eb3..013769395c 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -18,6 +18,7 @@ import ( ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { + recvMsg, err := consensus.ParseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). @@ -26,6 +27,9 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Msg("[OnAnnounce] Unparseable leader message") return } + if consensus.ShardID == 0 { + //fmt.Println("onAnnounce called ", recvMsg.BlockNum) + } // NOTE let it handle its own logs if !consensus.onAnnounceSanityChecks(recvMsg) { @@ -184,6 +188,9 @@ func (consensus *Consensus) sendCommitMessages(blockObj *types.Block) { // if onPrepared accepts the prepared message from the leader, then // it will send a COMMIT message for the leader to receive on the network. func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { + if consensus.ShardID == 0 { + //fmt.Println("onPrepared", recvMsg.BlockNum) + } consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -399,6 +406,13 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { consensus.getLogger().Info().Msg("[OnCommitted] Start consensus timer (new block added)") consensus.consensusTimeout[timeoutConsensus].Start() } + + //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum) + if blk != nil { + //consensus.ReshardingNextLeader(blk) + } else { + //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum, "blk is nil") + } } // Collect private keys that are part of the current committee. diff --git a/consensus/view_change.go b/consensus/view_change.go index 6631051dcc..fd9cde6c5f 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync" "time" @@ -161,6 +162,7 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { Uint64("stuckBlockViewID", stuckBlockViewID). Msg("[getNextViewID]") + fmt.Println("end getNextViewID: ", nextViewID, viewChangeDuration) // duration is always the fixed view change duration for synchronous view change return nextViewID, viewChangeDuration } @@ -233,6 +235,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe lastLeaderPubKey, gap) } + fmt.Println("wasfoundNext", consensus.Blockchain.Config().IsAllowlistEpoch(epoch), wasFound, next.Bytes.Hex(), lastLeaderPubKey.Bytes.Hex()) if !wasFound { consensus.getLogger().Warn(). Str("key", consensus.LeaderPubKey.Bytes.Hex()). @@ -254,6 +257,7 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { + fmt.Printf("Message to send leader111: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } @@ -264,6 +268,7 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() + //fmt.Println("startViewChange", nextViewID) consensus.SetViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet @@ -307,7 +312,9 @@ func (consensus *Consensus) startViewChange() { if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { continue } + // Тут уже другой leader msgToSend := consensus.constructViewChangeMessage(&key) + fmt.Println("Message to send leader222: ", consensus.LeaderPubKey.Bytes.Hex()) if err := consensus.msgSender.SendWithRetry( consensus.BlockNum(), msg_pb.MessageType_VIEWCHANGE, @@ -365,6 +372,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.ResetState() } + fmt.Println("[startNewView]", newLeaderPriKey.Pub.Bytes.Hex()) consensus.SetLeaderPubKey(newLeaderPriKey.Pub) return nil @@ -372,6 +380,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri // onViewChange is called when the view change message is received. func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { + //fmt.Printf("[onViewChange] received view change message from %+v\n", recvMsg) consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -394,6 +403,13 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { return } + consensus.getLogger().Debug(). + Err(err). + Interface("SenderPubkeys", recvMsg.SenderPubkeys). + Str("NextLeader", recvMsg.LeaderPubkey.Bytes.Hex()). + Str("myBLSPubKey", consensus.priKey.GetPublicKeys().SerializeToHexStr()). + Msg("[onViewChange] I am the Leader") + if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) { consensus.getLogger().Info(). Int64("have", consensus.Decider.SignersCount(quorum.ViewChange)). @@ -474,6 +490,8 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.mutex.Lock() defer consensus.mutex.Unlock() + fmt.Printf("[onNewView] received new view message from %+v\n", recvMsg) + consensus.getLogger().Info(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -562,6 +580,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // newView message verified success, override my state consensus.SetViewIDs(recvMsg.ViewID) consensus.pubKeyLock.Lock() + fmt.Println("[onNewView1221] new leader key cur:", consensus.LeaderPubKey.Bytes.Hex(), " new: ", senderKey.Bytes.Hex()) consensus.LeaderPubKey = senderKey consensus.pubKeyLock.Unlock() consensus.ResetViewChangeState() diff --git a/hmy/hmy.go b/hmy/hmy.go index 58e8d59edf..8089f04ba8 100644 --- a/hmy/hmy.go +++ b/hmy/hmy.go @@ -18,6 +18,7 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + "github.com/harmony-one/harmony/multibls" commonRPC "github.com/harmony-one/harmony/rpc/common" "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" @@ -96,6 +97,7 @@ type NodeAPI interface { GetStakingTransactionsCount(address, txType string) (uint64, error) GetTraceResultByHash(hash common.Hash) (json.RawMessage, error) IsCurrentlyLeader() bool + GetPublicKeys() multibls.PublicKeys IsOutOfSync(shardID uint32) bool SyncStatus(shardID uint32) (bool, uint64, uint64) SyncPeers() map[string]int diff --git a/node/api.go b/node/api.go index ceda968084..debcb201f6 100644 --- a/node/api.go +++ b/node/api.go @@ -6,6 +6,7 @@ import ( "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy" "github.com/harmony-one/harmony/internal/tikv" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/rosetta" hmy_rpc "github.com/harmony-one/harmony/rpc" rpc_common "github.com/harmony-one/harmony/rpc/common" @@ -18,6 +19,11 @@ func (node *Node) IsCurrentlyLeader() bool { return node.Consensus.IsLeader() } +// GetPublicKeys exposes if node is currently the leader node +func (node *Node) GetPublicKeys() multibls.PublicKeys { + return node.Consensus.GetPrivateKeys().GetPublicKeys() +} + // PeerConnectivity .. func (node *Node) PeerConnectivity() (int, int, int) { return node.host.PeerConnectivity() diff --git a/node/node_newblock.go b/node/node_newblock.go index 5050e4d6a7..cbc4ad05ff 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -2,6 +2,7 @@ package node import ( "errors" + "fmt" "sort" "strings" "time" @@ -88,6 +89,8 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp newBlock, err := node.ProposeNewBlock(newCommitSigsChan) if err == nil { + fmt.Printf("ProposeNewBlock: #%d :%d @%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), newBlock.Header().ViewID().Int64(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) + if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). Msg("Block with the same number was already proposed, abort.") @@ -145,6 +148,7 @@ func (node *Node) ProposeNewBlock(commitSigs chan []byte) (*types.Block, error) if node.Blockchain().Config().IsStaking(header.Epoch()) { blsPubKeyBytes := leaderKey.Object.GetAddress() coinbase.SetBytes(blsPubKeyBytes[:]) + fmt.Println("coinbase.SetBytes leader: ", leaderKey.Bytes.Hex(), coinbase.Hex()) } emptyAddr := common.Address{} diff --git a/test/configs/local-resharding.txt b/test/configs/local-resharding.txt index 4ce91ece1c..fc3d3e97b4 100644 --- a/test/configs/local-resharding.txt +++ b/test/configs/local-resharding.txt @@ -1,25 +1,12 @@ 127.0.0.1 9000 validator .hmy/65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204.key -127.0.0.1 9002 validator .hmy/40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809.key -127.0.0.1 9004 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key -127.0.0.1 9006 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key -127.0.0.1 9008 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key -127.0.0.1 9010 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key -127.0.0.1 9012 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key -127.0.0.1 9014 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key -127.0.0.1 9016 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key -127.0.0.1 9018 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key -127.0.0.1 9020 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key -127.0.0.1 9022 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key -127.0.0.1 9200 explorer null 0 +127.0.0.1 9002 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key +127.0.0.1 9004 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key +127.0.0.1 9006 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key +127.0.0.1 9008 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key +127.0.0.1 9010 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key +127.0.0.1 9099 explorer null 0 127.0.0.1 9100 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key -127.0.0.1 9102 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key -127.0.0.1 9104 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key -127.0.0.1 9106 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key -127.0.0.1 9108 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key -127.0.0.1 9110 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key -127.0.0.1 9112 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key -127.0.0.1 9114 validator .hmy/f47238daef97d60deedbde5302d05dea5de67608f11f406576e363661f7dcbc4a1385948549b31a6c70f6fde8a391486.key -127.0.0.1 9116 validator .hmy/fc4b9c535ee91f015efff3f32fbb9d32cdd9bfc8a837bb3eee89b8fff653c7af2050a4e147ebe5c7233dc2d5df06ee0a.key -127.0.0.1 9118 validator .hmy/ca86e551ee42adaaa6477322d7db869d3e203c00d7b86c82ebee629ad79cb6d57b8f3db28336778ec2180e56a8e07296.key -127.0.0.1 9300 explorer null 1 \ No newline at end of file +127.0.0.1 9102 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key +127.0.0.1 9104 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key +127.0.0.1 9106 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key \ No newline at end of file diff --git a/test/deploy.sh b/test/deploy.sh index 9ab6943110..fe22de57f0 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -69,7 +69,7 @@ function launch_localnet() { if ${VERBOSE}; then verbosity=5 else - verbosity=3 + verbosity=5 fi base_args=(--log_folder "${log_folder}" --min_peers "${MIN}" --bootnodes "${BN_MA}" "--network_type=$NETWORK" --blspass file:"${ROOT}/.hmy/blspass.txt" "--dns=false" "--verbosity=${verbosity}" "--p2p.security.max-conn-per-ip=100") @@ -80,8 +80,11 @@ function launch_localnet() { while IFS='' read -r line || [[ -n "$line" ]]; do i=$((i + 1)) + + # Read config for i-th node form config file IFS=' ' read -r ip port mode bls_key shard node_config <<<"${line}" + echo "LINE: ${line} ${shard}" args=("${base_args[@]}" --ip "${ip}" --port "${port}" --key "/tmp/${ip}-${port}.key" --db_dir "${ROOT}/db-${ip}-${port}" "--broadcast_invalid_tx=false") if [[ -z "$ip" || -z "$port" ]]; then echo "skip empty node" From 994283996f01b4da871b19e855af052c9785fb3d Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 5 Sep 2022 19:39:49 +0700 Subject: [PATCH 308/420] consensus check is forked --- api/service/explorer/service.go | 6 ++-- consensus/consensus.go | 2 -- consensus/consensus_service.go | 14 -------- consensus/consensus_v2.go | 61 ++++++++++++++++++++++++--------- 4 files changed, 47 insertions(+), 36 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index d96c1bc56c..5887bb3a02 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -223,9 +223,9 @@ func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { for i := cur; i > 0; i-- { block := s.blockchain.GetBlockByNumber(i) - w.Write([]byte(fmt.Sprintf("%d ", i))) - w.Write([]byte(fmt.Sprintf("%s ", block.Header().ViewID().String()))) - w.Write([]byte(fmt.Sprintf("%s ", block.Header().Coinbase().Hash().Hex()))) + w.Write([]byte(fmt.Sprintf("#%d ", i))) + w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) + w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) } } diff --git a/consensus/consensus.go b/consensus/consensus.go index 8477eb2aa9..a9fcc3cc52 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -71,8 +71,6 @@ type Consensus struct { priKey multibls.PrivateKeys // the publickey of leader LeaderPubKey *bls.PublicKeyWrapper - // index of leader in the list of validators. - LeaderIndex int // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e646504dde..d26ea8ced5 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -474,20 +474,6 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } -// SetLeaderIndex set the leader index. -func (consensus *Consensus) SetLeaderIndex(f func(int) int) (current int) { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() - consensus.LeaderIndex = f(consensus.LeaderIndex) - return consensus.LeaderIndex -} - -func (consensus *Consensus) GetLeaderIndex() int { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() - return consensus.LeaderIndex -} - // SetViewChangingID set the current view change ID func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index bf042c62d3..d92a417836 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -737,25 +737,41 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) // first view id is going to be 2. prev := consensus.GetLeaderPubKey() - idx := consensus.SetLeaderIndex(func(i int) int { - if curBlockViewID%3 == 0 { - return i + 1 + if consensus.Blockchain.Config().IsLeaderRotation(blk.Epoch()) { + epochBlockViewID, err := consensus.getEpochFirstBlockViewID(blk.Epoch()) + if err != nil { + consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", blk.Epoch().Uint64()) + return + } + if epochBlockViewID > curBlockViewID { + consensus.getLogger().Error().Msg("[SetupForNewConsensus] Epoch block viewID is greater than current block viewID") + return } - return i - }) - pps := consensus.Decider.Participants() - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = &pps[idx%len(pps)] - fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %q\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) - consensus.pubKeyLock.Unlock() - if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { - // leader changed - go func() { - fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - consensus.ReadySignal <- SyncProposal - }() + diff := curBlockViewID - epochBlockViewID + + pps := consensus.Decider.Participants() + idx := (int(diff) / 3) % len(pps) + consensus.pubKeyLock.Lock() + fmt.Println("(int(diff)/3)%len(pps) == ", idx) + consensus.LeaderPubKey = &pps[idx] + fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + consensus.pubKeyLock.Unlock() + if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { + // leader changed + go func() { + fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) + consensus.ReadySignal <- SyncProposal + + }() + } + } else { + fmt.Printf("SetupForNewConsensus0 :%d future v%d new: %s prev: %s %v\n", utils.GetPort(), curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] + consensus.pubKeyLock.Unlock() + } } var epoch *big.Int @@ -776,6 +792,17 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.ResetState() } +func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, error) { + if epoch.Uint64() == 0 { + return 0, nil + } + epochBlock := consensus.Blockchain.GetBlockByNumber(epoch.Uint64() - 1) + if epochBlock == nil { + return 0, errors.Errorf("block not found for number %d", epoch.Uint64()-1) + } + return epochBlock.Header().ViewID().Uint64() + 1, nil +} + func (consensus *Consensus) postCatchup(initBN uint64) { if initBN < consensus.BlockNum() { consensus.getLogger().Info(). From b71fc85ec0f4c05995ba884faf8a7e7409ce9426 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sat, 10 Sep 2022 18:02:45 +0700 Subject: [PATCH 309/420] fix --- api/service/explorer/service.go | 2 -- cmd/harmony/main.go | 3 --- consensus/consensus_service.go | 11 ----------- consensus/quorum/quorum.go | 7 +------ 4 files changed, 1 insertion(+), 22 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 5887bb3a02..6aa275332b 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -115,8 +115,6 @@ func (s *Service) Run() *http.Server { s.router = mux.NewRouter() - fmt.Println("++", addr) - // Set up router for addresses. // Fetch addresses request, accepts parameter size: how much addresses to read, // parameter prefix: from which address prefix start diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 0527a0b8c3..f62b81acd8 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -266,8 +266,6 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { var err error - fmt.Println("OS: ", os.Args) - nodeconfigSetShardSchedule(hc) nodeconfig.SetShardingSchedule(shard.Schedule) nodeconfig.SetVersion(getHarmonyVersion()) @@ -787,7 +785,6 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // Set the consensus ID to be the current block number viewID := currentNode.Blockchain().CurrentBlock().Header().ViewID().Uint64() - fmt.Println("viewID:", viewID) currentConsensus.SetViewIDs(viewID + 1) utils.Logger().Info(). Uint64("viewID", viewID). diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index d26ea8ced5..b010251217 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -213,7 +213,6 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } - fmt.Println("[checkViewID] Set LEADEER PUB KEY ", msg.SenderPubkeys[0].Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() @@ -397,7 +396,6 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { Str("leaderPubKey", leaderPubKey.Bytes.Hex()). Msg("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain") consensus.pubKeyLock.Lock() - fmt.Println("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain", leaderPubKey.Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = leaderPubKey consensus.pubKeyLock.Unlock() } @@ -484,15 +482,6 @@ func (consensus *Consensus) StartFinalityCount() { consensus.finalityCounter.Store(time.Now().UnixNano()) } -//func (consensus *Consensus) ReshardingNextLeader(newblock *types.Block) { -// consensus.pubKeyLock.Lock() -// fmt.Println("nextBlock1 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) -// consensus.LeaderPubKey = consensus.getNextLeaderKey(consensus.GetCurBlockViewID() + 1) -// fmt.Println("nextBlock2 ", newblock.Header().Number().Uint64(), " ", consensus.LeaderPubKey.Bytes.Hex()) -// consensus.pubKeyLock.Unlock() -// -//} - // FinishFinalityCount calculate the current finality func (consensus *Consensus) FinishFinalityCount() { d := time.Now().UnixNano() diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 6aa9b0dd79..da1551cdf9 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -229,17 +229,12 @@ func (s *cIdentities) NthNextHmy(instance shardingconfig.Instance, pubKey *bls.P Msg("[NthNextHmy] pubKey not found") } numNodes := instance.NumHarmonyOperatedNodesPerShard() - //fmt.Println("??idx:", idx, numNodes) // sanity check to avoid out of bound access if numNodes <= 0 || numNodes > len(s.publicKeys) { numNodes = len(s.publicKeys) } idx = (idx + next) % numNodes - //fmt.Println("-------idx:", idx) - - new := &s.publicKeys[idx] - fmt.Println("NthNextHmy: ", pubKey.Bytes.Hex(), new.Bytes.Hex()) - return found, new + return found, &s.publicKeys[idx] } // NthNextHmyExt return the Nth next pubkey of Harmony + allowlist nodes, next can be negative number From ab28f75be78de065af7001d08a6e58cfe8378456 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 15:27:56 +0700 Subject: [PATCH 310/420] Cleanup and fix update pub keys. --- cmd/harmony/main.go | 10 ++++++++++ consensus/consensus.go | 2 +- consensus/consensus_service.go | 4 ++++ consensus/consensus_v2.go | 1 - consensus/leader.go | 11 +++-------- consensus/view_change.go | 11 ----------- 6 files changed, 18 insertions(+), 21 deletions(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index f62b81acd8..8435f46e75 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -661,6 +661,16 @@ func createGlobalConfig(hc harmonyconfig.HarmonyConfig) (*nodeconfig.ConfigType, } func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfig.ConfigType, registry *registry.Registry) *node.Node { + // Consensus object. + // TODO: consensus object shouldn't start here + decider := quorum.NewDecider(quorum.SuperMajorityVote, uint32(hc.General.ShardID)) + currentConsensus, err := consensus.New( + myHost, nodeConfig.ShardID, p2p.Peer{}, nodeConfig.ConsensusPriKey, decider) + + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Error :%v \n", err) + os.Exit(1) + } // Parse minPeers from harmonyconfig.HarmonyConfig var minPeers int var aggregateSig bool diff --git a/consensus/consensus.go b/consensus/consensus.go index a9fcc3cc52..81ed237cf8 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -240,7 +240,7 @@ func (consensus *Consensus) getBlockNum() uint64 { // New create a new Consensus record func New( - host p2p.Host, shard uint32, multiBLSPriKey multibls.PrivateKeys, + host p2p.Host, shard uint32, leader p2p.Peer, multiBLSPriKey multibls.PrivateKeys, registry *registry.Registry, Decider quorum.Decider, minPeers int, aggregateSig bool, ) (*Consensus, error) { diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index b010251217..5055684e6c 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -74,6 +74,10 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { + if utils.GetPort() == 9000 { + //utils.Logger().Info().Msg("UpdatePublicKeys") + fmt.Println("UpdatePublicKeys", len(pubKeys), len(allowlist)) + } // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index d92a417836..6b2d6f7025 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -650,7 +650,6 @@ func (consensus *Consensus) tryCatchup() error { consensus.getLogger().Error().Err(err).Msg("[TryCatchup] Failed to add block to chain") return err } - //fmt.Println("tryCatchup ", utils.GetPort(), blk.NumberU64()) select { // TODO: Remove this when removing dns sync and stream sync is fully up case consensus.VerifiedNewBlock <- blk: diff --git a/consensus/leader.go b/consensus/leader.go index 21f3b770d5..0d9a65094d 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -191,16 +191,12 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { // TODO HERE - //if recvMsg.ViewID == 10 { - // return - //} + if recvMsg.ViewID == 21 { + return + } consensus.mutex.Lock() defer consensus.mutex.Unlock() //// Read - Start - if consensus.ShardID == 0 { - //fmt.Println("onCommit ", recvMsg.BlockNum) - } - if !consensus.isRightBlockNumAndViewID(recvMsg) { return } @@ -332,5 +328,4 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED) } - //fmt.Println("onCommit99: ", utils.GetPort(), recvMsg.BlockNum) } diff --git a/consensus/view_change.go b/consensus/view_change.go index fd9cde6c5f..3bd0c181f2 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync" "time" @@ -162,7 +161,6 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { Uint64("stuckBlockViewID", stuckBlockViewID). Msg("[getNextViewID]") - fmt.Println("end getNextViewID: ", nextViewID, viewChangeDuration) // duration is always the fixed view change duration for synchronous view change return nextViewID, viewChangeDuration } @@ -235,7 +233,6 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe lastLeaderPubKey, gap) } - fmt.Println("wasfoundNext", consensus.Blockchain.Config().IsAllowlistEpoch(epoch), wasFound, next.Bytes.Hex(), lastLeaderPubKey.Bytes.Hex()) if !wasFound { consensus.getLogger().Warn(). Str("key", consensus.LeaderPubKey.Bytes.Hex()). @@ -257,7 +254,6 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - fmt.Printf("Message to send leader111: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } @@ -268,7 +264,6 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() - //fmt.Println("startViewChange", nextViewID) consensus.SetViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet @@ -312,9 +307,7 @@ func (consensus *Consensus) startViewChange() { if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { continue } - // Тут уже другой leader msgToSend := consensus.constructViewChangeMessage(&key) - fmt.Println("Message to send leader222: ", consensus.LeaderPubKey.Bytes.Hex()) if err := consensus.msgSender.SendWithRetry( consensus.BlockNum(), msg_pb.MessageType_VIEWCHANGE, @@ -372,7 +365,6 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.ResetState() } - fmt.Println("[startNewView]", newLeaderPriKey.Pub.Bytes.Hex()) consensus.SetLeaderPubKey(newLeaderPriKey.Pub) return nil @@ -490,8 +482,6 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.mutex.Lock() defer consensus.mutex.Unlock() - fmt.Printf("[onNewView] received new view message from %+v\n", recvMsg) - consensus.getLogger().Info(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -580,7 +570,6 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // newView message verified success, override my state consensus.SetViewIDs(recvMsg.ViewID) consensus.pubKeyLock.Lock() - fmt.Println("[onNewView1221] new leader key cur:", consensus.LeaderPubKey.Bytes.Hex(), " new: ", senderKey.Bytes.Hex()) consensus.LeaderPubKey = senderKey consensus.pubKeyLock.Unlock() consensus.ResetViewChangeState() From bd1232703fce8303cee9f7d49a1ecb960e4ba855 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 13 Sep 2022 01:10:13 +0700 Subject: [PATCH 311/420] fix fix fix fix fix --- consensus/consensus_service.go | 5 ----- consensus/leader.go | 4 ---- node/node_newblock.go | 2 +- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 5055684e6c..1844f22687 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync/atomic" "time" @@ -74,10 +73,6 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { - if utils.GetPort() == 9000 { - //utils.Logger().Info().Msg("UpdatePublicKeys") - fmt.Println("UpdatePublicKeys", len(pubKeys), len(allowlist)) - } // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) diff --git a/consensus/leader.go b/consensus/leader.go index 0d9a65094d..8dbda2c589 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -190,10 +190,6 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { - // TODO HERE - if recvMsg.ViewID == 21 { - return - } consensus.mutex.Lock() defer consensus.mutex.Unlock() //// Read - Start diff --git a/node/node_newblock.go b/node/node_newblock.go index cbc4ad05ff..c29fb1b96a 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -89,7 +89,7 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp newBlock, err := node.ProposeNewBlock(newCommitSigsChan) if err == nil { - fmt.Printf("ProposeNewBlock: #%d :%d @%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), newBlock.Header().ViewID().Int64(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) + fmt.Printf("ProposeNewBlock: #%d :%d e:%d, sh:%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), node.Blockchain().CurrentHeader().Epoch().Uint64(), node.Blockchain().ShardID(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). From f3f7f036a0f417bd8b53ad45b505ea7a110a5ac2 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 17 Oct 2022 15:08:07 +0700 Subject: [PATCH 312/420] activate epoch --- internal/params/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/params/config.go b/internal/params/config.go index b50aa35532..fceca81627 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -111,6 +111,7 @@ var ( AllowlistEpoch: big.NewInt(2), LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 64, + TesnetNinetyPercentEpoch: big.NewInt(399), FeeCollectEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. From 0a12fc6f3620bfe3bf72ac28b096be7a24f791ff Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 17 Oct 2022 17:09:04 +0700 Subject: [PATCH 313/420] EpochTBD for leader rotation epoch. --- internal/params/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/params/config.go b/internal/params/config.go index fceca81627..56c5fe6609 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -110,7 +110,7 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(2), AllowlistEpoch: big.NewInt(2), LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 64, + LeaderRotationBlocksCount: 64, TesnetNinetyPercentEpoch: big.NewInt(399), FeeCollectEpoch: EpochTBD, } From 236cd86d43ab7cf4c01f048ed787166dd5a6a750 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 18 Oct 2022 12:17:17 +0700 Subject: [PATCH 314/420] 295 epoch --- internal/params/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/params/config.go b/internal/params/config.go index 56c5fe6609..fceca81627 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -110,7 +110,7 @@ var ( CrossShardXferPrecompileEpoch: big.NewInt(2), AllowlistEpoch: big.NewInt(2), LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 64, + LeaderRotationBlocksCount: 64, TesnetNinetyPercentEpoch: big.NewInt(399), FeeCollectEpoch: EpochTBD, } From 77a55bc0fff1adab26ef3c53698a7ec362723882 Mon Sep 17 00:00:00 2001 From: Konstantin <355847+Frozen@users.noreply.github.com> Date: Fri, 28 Oct 2022 12:52:36 +0300 Subject: [PATCH 315/420] Decider no longer requires public keys as a dependency. (#4289) --- cmd/harmony/main.go | 2 +- consensus/consensus.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 8435f46e75..b4b7d41cf7 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -665,7 +665,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // TODO: consensus object shouldn't start here decider := quorum.NewDecider(quorum.SuperMajorityVote, uint32(hc.General.ShardID)) currentConsensus, err := consensus.New( - myHost, nodeConfig.ShardID, p2p.Peer{}, nodeConfig.ConsensusPriKey, decider) + myHost, nodeConfig.ShardID, nodeConfig.ConsensusPriKey, decider) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "Error :%v \n", err) diff --git a/consensus/consensus.go b/consensus/consensus.go index 81ed237cf8..a9fcc3cc52 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -240,7 +240,7 @@ func (consensus *Consensus) getBlockNum() uint64 { // New create a new Consensus record func New( - host p2p.Host, shard uint32, leader p2p.Peer, multiBLSPriKey multibls.PrivateKeys, + host p2p.Host, shard uint32, multiBLSPriKey multibls.PrivateKeys, registry *registry.Registry, Decider quorum.Decider, minPeers int, aggregateSig bool, ) (*Consensus, error) { From f8903ada3b0b8a225b6dc4d3deea4331e4650566 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 19 Oct 2022 22:26:13 +0700 Subject: [PATCH 316/420] Consensus doesn't require anymore `Node` as a circular dependency. --- core/blockchain_impl.go | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 8b0683bd65..783a571e89 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -327,19 +327,8 @@ func VerifyBlockCrossLinks(blockchain BlockChain, block *types.Block) error { // ReadCrossLink beacon chain usage. cl, err := blockchain.ReadCrossLink(crossLink.ShardID(), crossLink.BlockNum()) if err == nil && cl != nil { - utils.Logger().Err(errAlreadyExist). - Uint64("beacon-block-number", block.NumberU64()). - Interface("remote", crossLink). - Interface("local", cl). - Msg("[CrossLinkVerification]") - // TODO Add slash for exist same blocknum but different crosslink - return errors.Wrapf( - errAlreadyExist, - "[CrossLinkVerification] shard: %d block: %d on beacon block %d", - crossLink.ShardID(), - crossLink.BlockNum(), - block.NumberU64(), - ) + // Add slash for exist same blocknum but different crosslink + return errAlreadyExist } if err := VerifyCrossLink(blockchain, crossLink); err != nil { return errors.Wrapf(err, "cannot VerifyBlockCrossLinks") From 66c6ee12878f498e51340af851e133bbc5595d71 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Fri, 28 Oct 2022 00:48:12 +0700 Subject: [PATCH 317/420] Proper blockchain initialization. --- cmd/harmony/main.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index b4b7d41cf7..f62b81acd8 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -661,16 +661,6 @@ func createGlobalConfig(hc harmonyconfig.HarmonyConfig) (*nodeconfig.ConfigType, } func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfig.ConfigType, registry *registry.Registry) *node.Node { - // Consensus object. - // TODO: consensus object shouldn't start here - decider := quorum.NewDecider(quorum.SuperMajorityVote, uint32(hc.General.ShardID)) - currentConsensus, err := consensus.New( - myHost, nodeConfig.ShardID, nodeConfig.ConsensusPriKey, decider) - - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "Error :%v \n", err) - os.Exit(1) - } // Parse minPeers from harmonyconfig.HarmonyConfig var minPeers int var aggregateSig bool From d20afa84fd8376b3d41d46fb000abd1c0f178ef4 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 00:33:16 +0800 Subject: [PATCH 318/420] Rwlock consensus. --- consensus/consensus_v2.go | 146 +++++++++++++++++++++----------------- 1 file changed, 80 insertions(+), 66 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 6b2d6f7025..d849898d60 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -292,7 +292,7 @@ func (consensus *Consensus) BlockCommitSigs(blockNum uint64) ([]byte, error) { // Start waits for the next new block and run consensus func (consensus *Consensus) Start( - blockChannel chan *types.Block, stopChan, stoppedChan, startChannel chan struct{}, + stopChan, stoppedChan, startChannel chan struct{}, ) { go func() { toStart := make(chan struct{}, 1) @@ -317,13 +317,13 @@ func (consensus *Consensus) Start( // Set up next block due time. consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) - start := false + consensus.start = false for { select { case <-toStart: - start = true + consensus.start = true case <-ticker.C: - if !start && isInitialLeader { + if !consensus.start && isInitialLeader { continue } for k, v := range consensus.consensusTimeout { @@ -362,68 +362,6 @@ func (consensus *Consensus) Start( } } - // TODO: Refactor this piece of code to consensus/downloader.go after DNS legacy sync is removed - case <-consensus.syncReadyChan: - consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncReadyChan") - consensus.mutex.Lock() - if consensus.BlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { - consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) - consensus.SetViewIDs(consensus.Blockchain().CurrentHeader().ViewID().Uint64() + 1) - mode := consensus.UpdateConsensusInformation() - consensus.current.SetMode(mode) - consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") - consensus.consensusTimeout[timeoutConsensus].Start() - consensus.getLogger().Info().Str("Mode", mode.String()).Msg("Node is IN SYNC") - consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() - } else if consensus.Mode() == Syncing { - // Corner case where sync is triggered before `onCommitted` and there is a race - // for block insertion between consensus and downloader. - mode := consensus.UpdateConsensusInformation() - consensus.SetMode(mode) - consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") - consensus.consensusTimeout[timeoutConsensus].Start() - consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() - } - consensus.mutex.Unlock() - - // TODO: Refactor this piece of code to consensus/downloader.go after DNS legacy sync is removed - case <-consensus.syncNotReadyChan: - consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncNotReadyChan") - consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) - consensus.current.SetMode(Syncing) - consensus.getLogger().Info().Msg("[ConsensusMainLoop] Node is OUT OF SYNC") - consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() - - case newBlock := <-blockChannel: - //consensus.ReshardingNextLeader(newBlock) - consensus.getLogger().Info(). - Uint64("MsgBlockNum", newBlock.NumberU64()). - Msg("[ConsensusMainLoop] Received Proposed New Block!") - - if newBlock.NumberU64() < consensus.BlockNum() { - consensus.getLogger().Warn().Uint64("newBlockNum", newBlock.NumberU64()). - Msg("[ConsensusMainLoop] received old block, abort") - continue - } - // Sleep to wait for the full block time - consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") - - <-time.After(time.Until(consensus.NextBlockDue)) - consensus.StartFinalityCount() - - // Update time due for next block - consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) - - startTime = time.Now() - consensus.msgSender.Reset(newBlock.NumberU64()) - - consensus.getLogger().Info(). - Int("numTxs", len(newBlock.Transactions())). - Int("numStakingTxs", len(newBlock.StakingTransactions())). - Time("startTime", startTime). - Int64("publicKeys", consensus.Decider.ParticipantsCount()). - Msg("[ConsensusMainLoop] STARTING CONSENSUS") - consensus.announce(newBlock) case <-stopChan: consensus.getLogger().Info().Msg("[ConsensusMainLoop] stopChan") return @@ -436,6 +374,82 @@ func (consensus *Consensus) Start( } } +func (consensus *Consensus) syncReadyChan() { + consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncReadyChan") + if consensus.BlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { + consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) + consensus.SetViewIDs(consensus.Blockchain().CurrentHeader().ViewID().Uint64() + 1) + mode := consensus.UpdateConsensusInformation() + consensus.current.SetMode(mode) + consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") + consensus.consensusTimeout[timeoutConsensus].Start() + consensus.getLogger().Info().Str("Mode", mode.String()).Msg("Node is IN SYNC") + consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() + } else if consensus.Mode() == Syncing { + // Corner case where sync is triggered before `onCommitted` and there is a race + // for block insertion between consensus and downloader. + mode := consensus.UpdateConsensusInformation() + consensus.SetMode(mode) + consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") + consensus.consensusTimeout[timeoutConsensus].Start() + consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() + } + +} + +func (consensus *Consensus) syncNotReadyChan() { + consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncNotReadyChan") + consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) + consensus.current.SetMode(Syncing) + consensus.getLogger().Info().Msg("[ConsensusMainLoop] Node is OUT OF SYNC") + consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() +} + +// Close close the consensus. If current is in normal commit phase, wait until the commit +// phase end. +func (consensus *Consensus) Close() error { + if consensus.dHelper != nil { + consensus.dHelper.close() + } + consensus.waitForCommit() + return nil +} + +func (consensus *Consensus) BlockChannel(newBlock *types.Block) { + //consensus.ReshardingNextLeader(newBlock)consensus.getLogger().Info(). + Uint64("MsgBlockNum", newBlock.NumberU64()). + Msg("[ConsensusMainLoop] Received Proposed New Block!") + + if newBlock.NumberU64() < consensus.BlockNum() { + consensus.getLogger().Warn().Uint64("newBlockNum", newBlock.NumberU64()). + Msg("[ConsensusMainLoop] received old block, abort") + return + } + // Sleep to wait for the full block time + consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") + time.AfterFunc(time.Until(consensus.NextBlockDue), func() { + consensus.StartFinalityCount() + + // Update time due for next block + consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) + + startTime = time.Now() + consensus.msgSender.Reset(newBlock.NumberU64()) + + consensus.getLogger().Info(). + Int("numTxs", len(newBlock.Transactions())). + Int("numStakingTxs", len(newBlock.StakingTransactions())). + Time("startTime", startTime). + Int64("publicKeys", consensus.Decider.ParticipantsCount()). + Msg("[ConsensusMainLoop] STARTING CONSENSUS") + consensus.announce(newBlock) + }) + + if consensus.dHelper != nil { + consensus.dHelper.start() + } +} + // Close closes the consensus. If current is in normal commit phase, wait until the commit // phase end. func (consensus *Consensus) Close() error { From 631edfe50d0f085f70746543fc56bdaf9fbffdf6 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 01:11:36 +0800 Subject: [PATCH 319/420] Removed channels. --- consensus/consensus_v2.go | 134 ++++++++++++++++++-------------------- 1 file changed, 65 insertions(+), 69 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index d849898d60..2a133755b7 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -292,81 +292,28 @@ func (consensus *Consensus) BlockCommitSigs(blockNum uint64) ([]byte, error) { // Start waits for the next new block and run consensus func (consensus *Consensus) Start( - stopChan, stoppedChan, startChannel chan struct{}, + stopChan chan struct{}, ) { go func() { - toStart := make(chan struct{}, 1) - isInitialLeader := consensus.IsLeader() - if isInitialLeader { - consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Waiting for consensus start") - // send a signal to indicate it's ready to run consensus - // this signal is consumed by node object to create a new block and in turn trigger a new consensus on it - go func() { - <-startChannel - toStart <- struct{}{} - consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Send ReadySignal") - consensus.ReadySignal <- SyncProposal - }() - } consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Consensus started") - defer close(stoppedChan) - ticker := time.NewTicker(250 * time.Millisecond) - defer ticker.Stop() + go func() { + ticker := time.NewTicker(250 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-stopChan: + return + case <-ticker.C: + consensus.tick() + } + } + }() + consensus.consensusTimeout[timeoutBootstrap].Start() consensus.getLogger().Info().Msg("[ConsensusMainLoop] Start bootstrap timeout (only once)") // Set up next block due time. consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) - consensus.start = false - for { - select { - case <-toStart: - consensus.start = true - case <-ticker.C: - if !consensus.start && isInitialLeader { - continue - } - for k, v := range consensus.consensusTimeout { - // stop timer in listening mode - if consensus.current.Mode() == Listening { - v.Stop() - continue - } - - if consensus.current.Mode() == Syncing { - // never stop bootstrap timer here in syncing mode as it only starts once - // if it is stopped, bootstrap will be stopped and nodes - // can't start view change or join consensus - // the bootstrap timer will be stopped once consensus is reached or view change - // is succeeded - if k != timeoutBootstrap { - consensus.getLogger().Debug(). - Str("k", k.String()). - Str("Mode", consensus.current.Mode().String()). - Msg("[ConsensusMainLoop] consensusTimeout stopped!!!") - v.Stop() - continue - } - } - if !v.CheckExpire() { - continue - } - if k != timeoutViewChange { - consensus.getLogger().Warn().Msg("[ConsensusMainLoop] Ops Consensus Timeout!!!") - consensus.startViewChange() - break - } else { - consensus.getLogger().Warn().Msg("[ConsensusMainLoop] Ops View Change Timeout!!!") - consensus.startViewChange() - break - } - } - - case <-stopChan: - consensus.getLogger().Info().Msg("[ConsensusMainLoop] stopChan") - return - } - } }() if consensus.dHelper != nil { @@ -374,6 +321,15 @@ func (consensus *Consensus) Start( } } +func (consensus *Consensus) StartChannel() { + consensus.isInitialLeader = consensus.IsLeader() + if consensus.isInitialLeader { + consensus.start = true + consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Send ReadySignal") + consensus.ReadySignal <- SyncProposal + } +} + func (consensus *Consensus) syncReadyChan() { consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncReadyChan") if consensus.BlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { @@ -394,7 +350,6 @@ func (consensus *Consensus) syncReadyChan() { consensus.consensusTimeout[timeoutConsensus].Start() consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() } - } func (consensus *Consensus) syncNotReadyChan() { @@ -405,7 +360,48 @@ func (consensus *Consensus) syncNotReadyChan() { consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() } -// Close close the consensus. If current is in normal commit phase, wait until the commit +func (consensus *Consensus) tick() { + if !consensus.start && consensus.isInitialLeader { + return + } + for k, v := range consensus.consensusTimeout { + // stop timer in listening mode + if consensus.current.Mode() == Listening { + v.Stop() + continue + } + + if consensus.current.Mode() == Syncing { + // never stop bootstrap timer here in syncing mode as it only starts once + // if it is stopped, bootstrap will be stopped and nodes + // can't start view change or join consensus + // the bootstrap timer will be stopped once consensus is reached or view change + // is succeeded + if k != timeoutBootstrap { + consensus.getLogger().Debug(). + Str("k", k.String()). + Str("Mode", consensus.current.Mode().String()). + Msg("[ConsensusMainLoop] consensusTimeout stopped!!!") + v.Stop() + continue + } + } + if !v.CheckExpire() { + continue + } + if k != timeoutViewChange { + consensus.getLogger().Warn().Msg("[ConsensusMainLoop] Ops Consensus Timeout!!!") + consensus.startViewChange() + break + } else { + consensus.getLogger().Warn().Msg("[ConsensusMainLoop] Ops View Change Timeout!!!") + consensus.startViewChange() + break + } + } +} + +// Close closes the consensus. If current is in normal commit phase, wait until the commit // phase end. func (consensus *Consensus) Close() error { if consensus.dHelper != nil { From b2e6ee57753fd94846daf39e8471bf482bbf6883 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 02:19:35 +0800 Subject: [PATCH 320/420] Removed view change locks. --- consensus/view_change.go | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/consensus/view_change.go b/consensus/view_change.go index 3bd0c181f2..0bfdd92215 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -2,7 +2,6 @@ package consensus import ( "math/big" - "sync" "time" "github.com/harmony-one/harmony/internal/chain" @@ -25,26 +24,21 @@ const MaxViewIDDiff = 249 // State contains current mode and current viewID type State struct { - mode Mode - modeMux sync.RWMutex + mode Mode // current view id in normal mode // it changes per successful consensus blockViewID uint64 - cViewMux sync.RWMutex // view changing id is used during view change mode // it is the next view id viewChangingID uint64 - viewMux sync.RWMutex isBackup bool } // Mode return the current node mode func (pm *State) Mode() Mode { - pm.modeMux.RLock() - defer pm.modeMux.RUnlock() return pm.mode } @@ -54,22 +48,16 @@ func (pm *State) SetMode(s Mode) { s = NormalBackup } - pm.modeMux.Lock() - defer pm.modeMux.Unlock() pm.mode = s } // GetCurBlockViewID return the current view id func (pm *State) GetCurBlockViewID() uint64 { - pm.cViewMux.RLock() - defer pm.cViewMux.RUnlock() return pm.blockViewID } // SetCurBlockViewID sets the current view id func (pm *State) SetCurBlockViewID(viewID uint64) uint64 { - pm.cViewMux.Lock() - defer pm.cViewMux.Unlock() pm.blockViewID = viewID return pm.blockViewID } @@ -77,26 +65,18 @@ func (pm *State) SetCurBlockViewID(viewID uint64) uint64 { // GetViewChangingID return the current view changing id // It is meaningful during view change mode func (pm *State) GetViewChangingID() uint64 { - pm.viewMux.RLock() - defer pm.viewMux.RUnlock() return pm.viewChangingID } // SetViewChangingID set the current view changing id // It is meaningful during view change mode func (pm *State) SetViewChangingID(id uint64) { - pm.viewMux.Lock() - defer pm.viewMux.Unlock() pm.viewChangingID = id } // GetViewChangeDuraion return the duration of the current view change // It increase in the power of difference betweeen view changing ID and current view ID func (pm *State) GetViewChangeDuraion() time.Duration { - pm.viewMux.RLock() - pm.cViewMux.RLock() - defer pm.viewMux.RUnlock() - defer pm.cViewMux.RUnlock() diff := int64(pm.viewChangingID - pm.blockViewID) return time.Duration(diff * diff * int64(viewChangeDuration)) } From cbcac648be2c46734fe332bea91092b46f1d9689 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 02:38:12 +0800 Subject: [PATCH 321/420] Removed multiSigMutex locks. --- consensus/consensus_service.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 1844f22687..5534b9bc93 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -138,9 +138,8 @@ func (consensus *Consensus) UpdateBitmaps() { multiSigBitmap, _ := bls_cosi.NewMask(members, nil) consensus.prepareBitmap = prepareBitmap consensus.commitBitmap = commitBitmap - consensus.multiSigMutex.Lock() consensus.multiSigBitmap = multiSigBitmap - consensus.multiSigMutex.Unlock() + } // ResetState resets the state of the consensus From a2b1fcdd7f499a740cd13610cfea44e1a550de3f Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 02:52:44 +0800 Subject: [PATCH 322/420] Removed leader locks. --- consensus/consensus_service.go | 7 ------ consensus/consensus_v2.go | 43 +++------------------------------- consensus/view_change.go | 7 +----- 3 files changed, 4 insertions(+), 53 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 5534b9bc93..3b46f54b7e 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -73,8 +73,6 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { - // TODO: use mutex for updating public keys pointer. No need to lock on all these logic. - consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) consensus.getLogger().Info().Msg("My Committee updated") for i := range pubKeys { @@ -93,7 +91,6 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi consensus.getLogger().Error(). Msg("[UpdatePublicKeys] Participants is empty") } - consensus.pubKeyLock.Unlock() // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. consensus.UpdateBitmaps() @@ -393,9 +390,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { consensus.getLogger().Info(). Str("leaderPubKey", leaderPubKey.Bytes.Hex()). Msg("[UpdateConsensusInformation] Most Recent LeaderPubKey Updated Based on BlockChain") - consensus.pubKeyLock.Lock() consensus.LeaderPubKey = leaderPubKey - consensus.pubKeyLock.Unlock() } } @@ -434,9 +429,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { - consensus.pubKeyLock.Lock() obj := consensus.LeaderPubKey.Object - consensus.pubKeyLock.Unlock() for _, key := range consensus.priKey { if key.Pub.Object.IsEqual(obj) { return true diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 2a133755b7..ad3436eaaa 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -744,52 +744,15 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { // SetupForNewConsensus sets the state for new consensus func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) - curBlockViewID := consensus.SetCurBlockViewID(committedMsg.ViewID + 1) // first view id is going to be 2. - prev := consensus.GetLeaderPubKey() - if consensus.Blockchain.Config().IsLeaderRotation(blk.Epoch()) { - epochBlockViewID, err := consensus.getEpochFirstBlockViewID(blk.Epoch()) - if err != nil { - consensus.getLogger().Error().Err(err).Msgf("[SetupForNewConsensus] Failed to get epoch block viewID for epoch %d", blk.Epoch().Uint64()) - return - } - if epochBlockViewID > curBlockViewID { - consensus.getLogger().Error().Msg("[SetupForNewConsensus] Epoch block viewID is greater than current block viewID") - return - } - - diff := curBlockViewID - epochBlockViewID - - pps := consensus.Decider.Participants() - idx := (int(diff) / 3) % len(pps) - consensus.pubKeyLock.Lock() - fmt.Println("(int(diff)/3)%len(pps) == ", idx) - consensus.LeaderPubKey = &pps[idx] - fmt.Printf("SetupForNewConsensus :%d idx: %d future v%d new: %s prev: %s %v\n", utils.GetPort(), idx, curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) - consensus.pubKeyLock.Unlock() - if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { - // leader changed - go func() { - fmt.Printf("ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - defer fmt.Printf("Defer ReadySignal :%d for leader %s\n", utils.GetPort(), consensus.GetLeaderPubKey().Bytes.Hex()) - consensus.ReadySignal <- SyncProposal - - }() - } - } else { - fmt.Printf("SetupForNewConsensus0 :%d future v%d new: %s prev: %s %v\n", utils.GetPort(), curBlockViewID, consensus.LeaderPubKey.Bytes.Hex(), prev.Bytes.Hex(), consensus.isLeader()) - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] - consensus.pubKeyLock.Unlock() - } - - } + consensus.SetCurBlockViewID(committedMsg.ViewID + 1) + consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] var epoch *big.Int if blk.IsLastBlockInEpoch() { epoch = new(big.Int).Add(blk.Epoch(), common.Big1) } else { epoch = blk.Epoch() } - if consensus.Blockchain().Config().IsLeaderRotation(epoch) { + if consensus.Blockchain.Config().IsLeaderRotation(epoch) { consensus.rotateLeader(epoch) } diff --git a/consensus/view_change.go b/consensus/view_change.go index 0bfdd92215..1267684d66 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -251,10 +251,7 @@ func (consensus *Consensus) startViewChange() { // aganist the consensus.LeaderPubKey variable. // Ideally, we shall use another variable to keep track of the // leader pubkey in viewchange mode - consensus.pubKeyLock.Lock() - lpk := consensus.getNextLeaderKey(nextViewID) - consensus.LeaderPubKey = lpk - consensus.pubKeyLock.Unlock() + consensus.LeaderPubKey = consensus.getNextLeaderKey(nextViewID) consensus.getLogger().Warn(). Uint64("nextViewID", nextViewID). @@ -549,9 +546,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // newView message verified success, override my state consensus.SetViewIDs(recvMsg.ViewID) - consensus.pubKeyLock.Lock() consensus.LeaderPubKey = senderKey - consensus.pubKeyLock.Unlock() consensus.ResetViewChangeState() consensus.msgSender.StopRetry(msg_pb.MessageType_VIEWCHANGE) From c83b126fa8461319bbc32c419db736f492ebeac4 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 16:51:24 +0800 Subject: [PATCH 323/420] Removed additional locks and isViewChange. --- consensus/consensus_service.go | 2 +- consensus/consensus_v2.go | 10 ++++++++-- consensus/validator.go | 4 ++-- consensus/view_change.go | 8 +++----- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 3b46f54b7e..ee0e6026c6 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -97,7 +97,7 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi consensus.ResetState() // do not reset view change state if it is in view changing mode - if !consensus.IsViewChangingMode() { + if !consensus.isViewChangingMode() { consensus.ResetViewChangeState() } return consensus.Decider.ParticipantsCount() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index ad3436eaaa..710c4e72b8 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -47,6 +47,12 @@ const ( // IsViewChangingMode return true if curernt mode is viewchanging func (consensus *Consensus) IsViewChangingMode() bool { + //consensus.mutex.RLock() + //defer consensus.mutex.RUnlock() + return consensus.isViewChangingMode() +} + +func (consensus *Consensus) isViewChangingMode() bool { return consensus.current.Mode() == ViewChanging } @@ -56,7 +62,7 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb // in order to avoid possible trap forever but drop PREPARE and COMMIT // which are message types specifically for a node acting as leader // so we just ignore those messages - if consensus.IsViewChangingMode() && + if consensus.isViewChangingMode() && (msg.Type == msg_pb.MessageType_PREPARE || msg.Type == msg_pb.MessageType_COMMIT) { return nil @@ -784,7 +790,7 @@ func (consensus *Consensus) postCatchup(initBN uint64) { consensus.switchPhase("TryCatchup", FBFTAnnounce) } // catch up and skip from view change trap - if initBN < consensus.BlockNum() && consensus.IsViewChangingMode() { + if initBN < consensus.BlockNum() && consensus.isViewChangingMode() { consensus.current.SetMode(Normal) consensus.consensusTimeout[timeoutViewChange].Stop() } diff --git a/consensus/validator.go b/consensus/validator.go index 013769395c..3e958fef26 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -47,7 +47,7 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { consensus.blockHash = recvMsg.BlockHash // we have already added message and block, skip check viewID // and send prepare message if is in ViewChanging mode - if consensus.IsViewChangingMode() { + if consensus.isViewChangingMode() { consensus.getLogger().Debug(). Msg("[OnAnnounce] Still in ViewChanging Mode, Exiting !!") return @@ -392,7 +392,7 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { return } - if consensus.IsViewChangingMode() { + if consensus.isViewChangingMode() { consensus.getLogger().Info().Msg("[OnCommitted] Still in ViewChanging mode, Exiting!!") return } diff --git a/consensus/view_change.go b/consensus/view_change.go index 1267684d66..aff25c2d15 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -237,8 +237,6 @@ func (consensus *Consensus) startViewChange() { if consensus.disableViewChange || consensus.IsBackup() { return } - consensus.mutex.Lock() - defer consensus.mutex.Unlock() consensus.consensusTimeout[timeoutConsensus].Stop() consensus.consensusTimeout[timeoutBootstrap].Stop() @@ -300,7 +298,7 @@ func (consensus *Consensus) startViewChange() { // startNewView stops the current view change func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.PrivateKeyWrapper, reset bool) error { - if !consensus.IsViewChangingMode() { + if !consensus.isViewChangingMode() { return errors.New("not in view changing mode anymore") } @@ -421,7 +419,7 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { } // received enough view change messages, change state to normal consensus - if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) && consensus.IsViewChangingMode() { + if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) && consensus.isViewChangingMode() { // no previous prepared message, go straight to normal mode // and start proposing new block if consensus.vc.IsM1PayloadEmpty() { @@ -537,7 +535,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { } } - if !consensus.IsViewChangingMode() { + if !consensus.isViewChangingMode() { consensus.getLogger().Info().Msg("Not in ViewChanging Mode.") return } From 50f1c78944650181735b545bf31939cb36182ffc Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 18:37:20 +0800 Subject: [PATCH 324/420] Added locks detected by race. --- consensus/consensus_v2.go | 5 ++++- scripts/go_executable_build.sh | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 710c4e72b8..4575e197ef 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -310,16 +310,19 @@ func (consensus *Consensus) Start( case <-stopChan: return case <-ticker.C: + consensus.mutex.Lock() consensus.tick() + consensus.mutex.Unlock() } } }() + consensus.mutex.Lock() consensus.consensusTimeout[timeoutBootstrap].Start() consensus.getLogger().Info().Msg("[ConsensusMainLoop] Start bootstrap timeout (only once)") - // Set up next block due time. consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) + consensus.mutex.Unlock() }() if consensus.dHelper != nil { diff --git a/scripts/go_executable_build.sh b/scripts/go_executable_build.sh index 6331aa5225..9d111d79f8 100755 --- a/scripts/go_executable_build.sh +++ b/scripts/go_executable_build.sh @@ -120,6 +120,7 @@ function build_only if [ "$STATIC" == "true" ]; then env GOOS=$GOOS GOARCH=$GOARCH go build $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY} -w -extldflags \"-static -lm\"" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} else +# env GOOS=$GOOS GOARCH=$GOARCH go build -race $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY}" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} env GOOS=$GOOS GOARCH=$GOARCH go build $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY}" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} fi fi From 7db3134249073dff5ffb42acfae2d4a4c1df2541 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 18:37:42 +0800 Subject: [PATCH 325/420] Added locks detected by race. --- scripts/go_executable_build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/go_executable_build.sh b/scripts/go_executable_build.sh index 9d111d79f8..6331aa5225 100755 --- a/scripts/go_executable_build.sh +++ b/scripts/go_executable_build.sh @@ -120,7 +120,6 @@ function build_only if [ "$STATIC" == "true" ]; then env GOOS=$GOOS GOARCH=$GOARCH go build $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY} -w -extldflags \"-static -lm\"" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} else -# env GOOS=$GOOS GOARCH=$GOARCH go build -race $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY}" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} env GOOS=$GOOS GOARCH=$GOARCH go build $VERBOSE -gcflags="${GO_GCFLAGS}" -ldflags="-X main.version=v${VERSION} -X main.commit=${COMMIT} -X main.builtAt=${BUILTAT} -X main.builtBy=${BUILTBY}" -o $BINDIR/$bin $RACE $TRACEPTR ${SRC[$bin]} fi fi From 5c7340c2e2677f1d6a35221a5ab05d82d0f130fe Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 19:38:31 +0800 Subject: [PATCH 326/420] Locks for start. --- consensus/consensus_v2.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 4575e197ef..4db16a3725 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -331,12 +331,16 @@ func (consensus *Consensus) Start( } func (consensus *Consensus) StartChannel() { + consensus.mutex.Lock() consensus.isInitialLeader = consensus.IsLeader() if consensus.isInitialLeader { consensus.start = true consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Send ReadySignal") + consensus.mutex.Unlock() consensus.ReadySignal <- SyncProposal + return } + consensus.mutex.Unlock() } func (consensus *Consensus) syncReadyChan() { From a0e8b191fbfc140d3f3d7be7d18a97d0d3f537a9 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 2 Nov 2022 20:35:59 +0800 Subject: [PATCH 327/420] Removed additional locks. --- consensus/consensus_service.go | 1 - consensus/consensus_v2.go | 2 -- 2 files changed, 3 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index ee0e6026c6..9a171ea1d0 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -493,7 +493,6 @@ func (consensus *Consensus) switchPhase(subject string, desired FBFTPhase) { Str("switchPhase:", subject) consensus.phase.Set(desired) - return } var ( diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 4db16a3725..876613d819 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -47,8 +47,6 @@ const ( // IsViewChangingMode return true if curernt mode is viewchanging func (consensus *Consensus) IsViewChangingMode() bool { - //consensus.mutex.RLock() - //defer consensus.mutex.RUnlock() return consensus.isViewChangingMode() } From 3db9efb8d43c21365fa01a26bb016451292517d5 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 3 Nov 2022 01:58:08 +0800 Subject: [PATCH 328/420] Removed additional locks. --- consensus/consensus_service.go | 2 +- consensus/validator.go | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 9a171ea1d0..aec2e2b94b 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -396,7 +396,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { for _, key := range pubKeys { // in committee - myPubKeys := consensus.GetPublicKeys() + myPubKeys := consensus.getPublicKeys() if myPubKeys.Contains(key.Object) { if hasError { consensus.getLogger().Error(). diff --git a/consensus/validator.go b/consensus/validator.go index 3e958fef26..ffdcedb354 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -79,10 +79,6 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { } func (consensus *Consensus) validateNewBlock(recvMsg *FBFTMessage) (*types.Block, error) { - // Lock to prevent race condition between announce and prepare - consensus.verifyBlockMutex.Lock() - defer consensus.verifyBlockMutex.Unlock() - if consensus.FBFTLog.IsBlockVerified(recvMsg.BlockHash) { var blockObj *types.Block @@ -188,12 +184,6 @@ func (consensus *Consensus) sendCommitMessages(blockObj *types.Block) { // if onPrepared accepts the prepared message from the leader, then // it will send a COMMIT message for the leader to receive on the network. func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { - if consensus.ShardID == 0 { - //fmt.Println("onPrepared", recvMsg.BlockNum) - } - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - consensus.getLogger().Info(). Uint64("MsgBlockNum", recvMsg.BlockNum). Uint64("MsgViewID", recvMsg.ViewID). From d39fb7f5e5ebd894cc29aee8858d133163c65b2f Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sun, 6 Nov 2022 22:47:14 +0700 Subject: [PATCH 329/420] Make func private. --- consensus/consensus_service.go | 6 +++--- consensus/consensus_v2.go | 2 +- consensus/view_change.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index aec2e2b94b..bf420177cc 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -94,7 +94,7 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. consensus.UpdateBitmaps() - consensus.ResetState() + consensus.resetState() // do not reset view change state if it is in view changing mode if !consensus.isViewChangingMode() { @@ -140,7 +140,7 @@ func (consensus *Consensus) UpdateBitmaps() { } // ResetState resets the state of the consensus -func (consensus *Consensus) ResetState() { +func (consensus *Consensus) resetState() { consensus.switchPhase("ResetState", FBFTAnnounce) consensus.blockHash = [32]byte{} @@ -520,7 +520,7 @@ func (consensus *Consensus) selfCommit(payload []byte) error { } // Have to keep the block hash so the leader can finish the commit phase of prepared block - consensus.ResetState() + consensus.resetState() copy(consensus.blockHash[:], blockHash[:]) consensus.switchPhase("selfCommit", FBFTCommit) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 876613d819..763214c035 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -772,7 +772,7 @@ func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg consensus.SetMode(consensus.UpdateConsensusInformation()) } consensus.FBFTLog.PruneCacheBeforeBlock(blk.NumberU64()) - consensus.ResetState() + consensus.resetState() } func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, error) { diff --git a/consensus/view_change.go b/consensus/view_change.go index aff25c2d15..95ccbb97aa 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -338,7 +338,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri // TODO: consider make ResetState unified and only called in one place like finalizeCommit() if reset { - consensus.ResetState() + consensus.resetState() } consensus.SetLeaderPubKey(newLeaderPriKey.Pub) @@ -554,7 +554,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.sendCommitMessages(preparedBlock) consensus.switchPhase("onNewView", FBFTCommit) } else { - consensus.ResetState() + consensus.resetState() consensus.getLogger().Info().Msg("onNewView === announce") } consensus.getLogger().Info(). From 2621d31ca52755d522749cbabf439d1c0b50a06b Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sun, 6 Nov 2022 23:42:26 +0700 Subject: [PATCH 330/420] Make VerifyBlock private. --- consensus/consensus_v2.go | 2 +- consensus/validator.go | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 763214c035..18feeae286 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -662,7 +662,7 @@ func (consensus *Consensus) tryCatchup() error { } blk.SetCurrentCommitSig(msg.Payload) - if err := consensus.VerifyBlock(blk); err != nil { + if err := consensus.verifyBlock(blk); err != nil { consensus.getLogger().Err(err).Msg("[TryCatchup] failed block verifier") return err } diff --git a/consensus/validator.go b/consensus/validator.go index ffdcedb354..b33ac6d688 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -42,8 +42,6 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Uint64("MsgBlockNum", recvMsg.BlockNum). Msg("[OnAnnounce] Announce message Added") consensus.FBFTLog.AddVerifiedMessage(recvMsg) - consensus.mutex.Lock() - defer consensus.mutex.Unlock() consensus.blockHash = recvMsg.BlockHash // we have already added message and block, skip check viewID // and send prepare message if is in ViewChanging mode @@ -78,6 +76,11 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { } } +func (consensus *Consensus) ValidateNewBlock(recvMsg *FBFTMessage) (*types.Block, error) { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + return consensus.validateNewBlock(recvMsg) +} func (consensus *Consensus) validateNewBlock(recvMsg *FBFTMessage) (*types.Block, error) { if consensus.FBFTLog.IsBlockVerified(recvMsg.BlockHash) { var blockObj *types.Block @@ -132,7 +135,7 @@ func (consensus *Consensus) validateNewBlock(recvMsg *FBFTMessage) (*types.Block return nil, errors.New("nil block verifier") } - if err := consensus.VerifyBlock(&blockObj); err != nil { + if err := consensus.verifyBlock(&blockObj); err != nil { consensus.getLogger().Error().Err(err).Msg("[validateNewBlock] Block verification failed") return nil, errors.New("Block verification failed") } From ab919eb534979ab160b13f887d5ab906623f8d48 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sun, 6 Nov 2022 23:55:22 +0700 Subject: [PATCH 331/420] Make IsLeader private. --- consensus/consensus_service.go | 11 ++++++++++- consensus/consensus_v2.go | 10 +++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index bf420177cc..fdbcbc0457 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -408,7 +408,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // If the leader changed and I myself become the leader if (oldLeader != nil && consensus.LeaderPubKey != nil && - !consensus.LeaderPubKey.Object.IsEqual(oldLeader.Object)) && consensus.IsLeader() { + !consensus.LeaderPubKey.Object.IsEqual(oldLeader.Object)) && consensus.isLeader() { go func() { consensus.getLogger().Info(). Str("myKey", myPubKeys.SerializeToHexStr()). @@ -429,6 +429,15 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() + + return consensus.isLeader() +} + +// IsLeader check if the node is a leader or not by comparing the public key of +// the node with the leader public key +func (consensus *Consensus) isLeader() bool { obj := consensus.LeaderPubKey.Object for _, key := range consensus.priKey { if key.Pub.Object.IsEqual(obj) { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 18feeae286..1fc26e7607 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -100,8 +100,8 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb canHandleViewChange := true intendedForValidator, intendedForLeader := - !consensus.IsLeader(), - consensus.IsLeader() + !consensus.isLeader(), + consensus.isLeader() // if in backup normal mode, force ignore view change event and leader event. if consensus.current.Mode() == NormalBackup { @@ -183,7 +183,7 @@ func (consensus *Consensus) finalCommit() { // Note: leader already sent 67% commit in preCommit. The 100% commit won't be sent immediately // to save network traffic. It will only be sent in retry if consensus doesn't move forward. // Or if the leader is changed for next block, the 100% committed sig will be sent to the next leader immediately. - if !consensus.IsLeader() || block.IsLastBlockInEpoch() { + if !consensus.isLeader() || block.IsLastBlockInEpoch() { // send immediately if err := consensus.msgSender.SendWithRetry( block.NumberU64(), @@ -248,7 +248,7 @@ func (consensus *Consensus) finalCommit() { // If still the leader, send commit sig/bitmap to finish the new block proposal, // else, the block proposal will timeout by itself. - if consensus.IsLeader() { + if consensus.isLeader() { if block.IsLastBlockInEpoch() { // No pipelining go func() { @@ -330,7 +330,7 @@ func (consensus *Consensus) Start( func (consensus *Consensus) StartChannel() { consensus.mutex.Lock() - consensus.isInitialLeader = consensus.IsLeader() + consensus.isInitialLeader = consensus.isLeader() if consensus.isInitialLeader { consensus.start = true consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Send ReadySignal") From ea0b97be646a7c9c59b6d8a0b80aa0960171a374 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 00:14:19 +0700 Subject: [PATCH 332/420] Make ParseFBFTMessage private. --- consensus/consensus_v2.go | 2 +- consensus/validator.go | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 1fc26e7607..77beac988c 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -92,7 +92,7 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb members := consensus.Decider.Participants() fbftMsg, err = ParseNewViewMessage(msg, members) default: - fbftMsg, err = consensus.ParseFBFTMessage(msg) + fbftMsg, err = consensus.parseFBFTMessage(msg) } if err != nil || fbftMsg == nil { return errors.Wrapf(err, "unable to parse consensus msg with type: %s", msg.Type) diff --git a/consensus/validator.go b/consensus/validator.go index b33ac6d688..cb2971e4cf 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -18,8 +18,7 @@ import ( ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { - - recvMsg, err := consensus.ParseFBFTMessage(msg) + recvMsg, err := consensus.parseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). Err(err). @@ -27,9 +26,6 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Msg("[OnAnnounce] Unparseable leader message") return } - if consensus.ShardID == 0 { - //fmt.Println("onAnnounce called ", recvMsg.BlockNum) - } // NOTE let it handle its own logs if !consensus.onAnnounceSanityChecks(recvMsg) { From 0ec4c81121a885764aeb754fe8ed4b10e557eea8 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 11:46:49 +0700 Subject: [PATCH 333/420] Fix remove locks. --- consensus/consensus_service.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index fdbcbc0457..4784b99fc0 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -429,8 +429,9 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { - consensus.mutex.RLock() - defer consensus.mutex.RUnlock() + // TODO: if remove locks blockchain stucks. + //consensus.mutex.RLock() + //defer consensus.mutex.RUnlock() return consensus.isLeader() } From 9328a0aea96356b83a70082c4806bec96931dae3 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 11:58:01 +0700 Subject: [PATCH 334/420] Added additional locks. --- consensus/consensus_v2.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 77beac988c..a862d725ab 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -436,7 +436,8 @@ func (consensus *Consensus) BlockChannel(newBlock *types.Block) { consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") time.AfterFunc(time.Until(consensus.NextBlockDue), func() { consensus.StartFinalityCount() - + consensus.mutex.Lock() + defer consensus.mutex.Unlock() // Update time due for next block consensus.NextBlockDue = time.Now().Add(consensus.BlockPeriod) From cb78d53c88849e9d08609a4462930cf401e95b1f Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 12:05:41 +0700 Subject: [PATCH 335/420] Added additional locks. --- consensus/validator.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/consensus/validator.go b/consensus/validator.go index cb2971e4cf..79e988fce4 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -271,6 +271,8 @@ func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { return } curBlockNum := consensus.BlockNum() + consensus.mutex.Lock() + defer consensus.mutex.Unlock() for _, committedMsg := range consensus.FBFTLog.GetNotVerifiedCommittedMessages(blockObj.NumberU64(), blockObj.Header().ViewID().Uint64(), blockObj.Hash()) { if committedMsg != nil { consensus.onCommitted(committedMsg) @@ -284,9 +286,6 @@ func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - consensus.getLogger().Info(). Uint64("MsgBlockNum", recvMsg.BlockNum). Uint64("MsgViewID", recvMsg.ViewID). From e7b3b175c06797e5daf276d21fd5b6de200d938f Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 12:22:26 +0700 Subject: [PATCH 336/420] Added readSignatureBitmapPayload locks. --- consensus/consensus_service.go | 15 ++++++++++----- consensus/validator.go | 4 ++-- consensus/view_change.go | 2 +- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 4784b99fc0..53a1a1a7c0 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -7,6 +7,7 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/crypto/bls" + "github.com/harmony-one/harmony/multibls" "github.com/ethereum/go-ethereum/common" protobuf "github.com/golang/protobuf/proto" @@ -229,16 +230,20 @@ func (consensus *Consensus) SetBlockNum(blockNum uint64) { } // ReadSignatureBitmapPayload read the payload for signature and bitmap; offset is the beginning position of reading -func (consensus *Consensus) ReadSignatureBitmapPayload( - recvPayload []byte, offset int, -) (*bls_core.Sign, *bls_cosi.Mask, error) { +func (consensus *Consensus) ReadSignatureBitmapPayload(recvPayload []byte, offset int) (*bls_core.Sign, *bls_cosi.Mask, error) { + consensus.mutex.RLock() + members := consensus.Decider.Participants() + consensus.mutex.RUnlock() + return consensus.readSignatureBitmapPayload(recvPayload, offset, members) +} + +func (consensus *Consensus) readSignatureBitmapPayload(recvPayload []byte, offset int, members multibls.PublicKeys) (*bls_core.Sign, *bls_cosi.Mask, error) { if offset+bls.BLSSignatureSizeInBytes > len(recvPayload) { return nil, nil, errors.New("payload not have enough length") } sigAndBitmapPayload := recvPayload[offset:] // TODO(audit): keep a Mask in the Decider so it won't be reconstructed on the fly. - members := consensus.Decider.Participants() return chain.ReadSignatureBitmapByPublicKeys( sigAndBitmapPayload, members, ) @@ -524,7 +529,7 @@ func (consensus *Consensus) selfCommit(payload []byte) error { return errGetPreparedBlock } - aggSig, mask, err := consensus.ReadSignatureBitmapPayload(payload, 32) + aggSig, mask, err := consensus.readSignatureBitmapPayload(payload, 32, consensus.Decider.Participants()) if err != nil { return errReadBitmapPayload } diff --git a/consensus/validator.go b/consensus/validator.go index 79e988fce4..7336c7b477 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -62,7 +62,7 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { if len(recvMsg.Block) > 0 { go func() { // Best effort check, no need to error out. - _, err := consensus.validateNewBlock(recvMsg) + _, err := consensus.ValidateNewBlock(recvMsg) if err == nil { consensus.getLogger().Info(). @@ -205,7 +205,7 @@ func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { // check validity of prepared signature blockHash := recvMsg.BlockHash - aggSig, mask, err := consensus.ReadSignatureBitmapPayload(recvMsg.Payload, 0) + aggSig, mask, err := consensus.readSignatureBitmapPayload(recvMsg.Payload, 0, consensus.Decider.Participants()) if err != nil { consensus.getLogger().Error().Err(err).Msg("ReadSignatureBitmapPayload failed!") return diff --git a/consensus/view_change.go b/consensus/view_change.go index 95ccbb97aa..9951cf06ba 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -501,7 +501,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { utils.CountOneBits(m3Mask.Bitmap) > utils.CountOneBits(m2Mask.Bitmap)) { // m1 is not empty, check it's valid blockHash := recvMsg.Payload[:32] - aggSig, mask, err := consensus.ReadSignatureBitmapPayload(recvMsg.Payload, 32) + aggSig, mask, err := consensus.readSignatureBitmapPayload(recvMsg.Payload, 32, consensus.Decider.Participants()) if err != nil { consensus.getLogger().Error().Err(err). Msg("[onNewView] ReadSignatureBitmapPayload Failed") From 0369740c10e542d52e8b336238b3450619513e64 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 12:50:03 +0700 Subject: [PATCH 337/420] Added HandleMessageUpdate locks. --- consensus/consensus_service.go | 7 ++++++- consensus/consensus_v2.go | 8 +++++--- consensus/leader.go | 2 -- consensus/view_change.go | 11 ++--------- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 53a1a1a7c0..d613f70004 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -468,7 +468,12 @@ func (consensus *Consensus) isLeader() bool { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { - fmt.Println("SetViewIDs", height) + consensus.setViewIDs(height) +} + +// SetViewIDs set both current view ID and view changing ID to the height +// of the blockchain. It is used during client startup to recover the state +func (consensus *Consensus) setViewIDs(height uint64) { consensus.SetCurBlockViewID(height) consensus.SetViewChangingID(height) } diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index a862d725ab..e866e1af71 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -56,6 +56,8 @@ func (consensus *Consensus) isViewChangingMode() bool { // HandleMessageUpdate will update the consensus state according to received message func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb.Message, senderKey *bls.SerializedPublicKey) error { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() // when node is in ViewChanging mode, it still accepts normal messages into FBFTLog // in order to avoid possible trap forever but drop PREPARE and COMMIT // which are message types specifically for a node acting as leader @@ -144,7 +146,7 @@ func (consensus *Consensus) finalCommit() { Msg("[finalCommit] Finalizing Consensus") beforeCatchupNum := consensus.BlockNum() - leaderPriKey, err := consensus.GetConsensusLeaderPrivateKey() + leaderPriKey, err := consensus.getConsensusLeaderPrivateKey() if err != nil { consensus.getLogger().Error().Err(err).Msg("[finalCommit] leader not found") return @@ -559,7 +561,7 @@ func (consensus *Consensus) preCommitAndPropose(blk *types.Block) error { return errors.New("block to pre-commit is nil") } - leaderPriKey, err := consensus.GetConsensusLeaderPrivateKey() + leaderPriKey, err := consensus.getConsensusLeaderPrivateKey() if err != nil { consensus.getLogger().Error().Err(err).Msg("[preCommitAndPropose] leader not found") return err @@ -804,7 +806,7 @@ func (consensus *Consensus) postCatchup(initBN uint64) { // GenerateVrfAndProof generates new VRF/Proof from hash of previous block func (consensus *Consensus) GenerateVrfAndProof(newHeader *block.Header) error { - key, err := consensus.GetConsensusLeaderPrivateKey() + key, err := consensus.getConsensusLeaderPrivateKey() if err != nil { return errors.New("[GenerateVrfAndProof] no leader private key provided") } diff --git a/consensus/leader.go b/consensus/leader.go index 8dbda2c589..2f7766e19b 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -190,8 +190,6 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { } func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { - consensus.mutex.Lock() - defer consensus.mutex.Unlock() //// Read - Start if !consensus.isRightBlockNumAndViewID(recvMsg) { return diff --git a/consensus/view_change.go b/consensus/view_change.go index 9951cf06ba..720205ee83 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -347,10 +347,6 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri // onViewChange is called when the view change message is received. func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { - //fmt.Printf("[onViewChange] received view change message from %+v\n", recvMsg) - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - consensus.getLogger().Debug(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -359,7 +355,7 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { // if not leader, noop newLeaderKey := recvMsg.LeaderPubkey - newLeaderPriKey, err := consensus.GetLeaderPrivateKey(newLeaderKey.Object) + newLeaderPriKey, err := consensus.getLeaderPrivateKey(newLeaderKey.Object) if err != nil { consensus.getLogger().Debug(). Err(err). @@ -454,9 +450,6 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { // Or the validator will enter announce phase to wait for the new block proposed // from the new leader func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - consensus.getLogger().Info(). Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). @@ -543,7 +536,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { consensus.consensusTimeout[timeoutViewChange].Stop() // newView message verified success, override my state - consensus.SetViewIDs(recvMsg.ViewID) + consensus.setViewIDs(recvMsg.ViewID) consensus.LeaderPubKey = senderKey consensus.ResetViewChangeState() From c2e1692d4ba2dc9499c70415810cde728df3991f Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 13:08:15 +0700 Subject: [PATCH 338/420] Added LastMile locks. --- consensus/consensus_v2.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index e866e1af71..4253a46e95 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -496,24 +496,24 @@ type LastMileBlockIter struct { } // GetLastMileBlockIter get the iterator of the last mile blocks starting from number bnStart -func (consensus *Consensus) GetLastMileBlockIter(bnStart uint64) (*LastMileBlockIter, error) { +func (consensus *Consensus) GetLastMileBlockIter(bnStart uint64, cb func(iter *LastMileBlockIter) error) error { consensus.mutex.Lock() defer consensus.mutex.Unlock() if consensus.BlockVerifier == nil { - return nil, errors.New("consensus haven't initialized yet") + return errors.New("consensus haven't initialized yet") } blocks, _, err := consensus.getLastMileBlocksAndMsg(bnStart) if err != nil { - return nil, err + return err } - return &LastMileBlockIter{ + return cb(&LastMileBlockIter{ blockCandidates: blocks, fbftLog: consensus.FBFTLog, verify: consensus.BlockVerifier, curIndex: 0, logger: consensus.getLogger(), - }, nil + }) } // Next iterate to the next last mile block From e08ce0a0a8bf3e6c68a7bc6991223b2eb5dd39ba Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 17:23:59 +0700 Subject: [PATCH 339/420] Locks for IsValidatorInCommittee. --- consensus/consensus_service.go | 6 ++++++ consensus/validator.go | 2 +- consensus/view_change.go | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index d613f70004..fdf0880625 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -159,6 +159,12 @@ func (consensus *Consensus) resetState() { // IsValidatorInCommittee returns whether the given validator BLS address is part of my committee func (consensus *Consensus) IsValidatorInCommittee(pubKey bls.SerializedPublicKey) bool { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() + return consensus.isValidatorInCommittee(pubKey) +} + +func (consensus *Consensus) isValidatorInCommittee(pubKey bls.SerializedPublicKey) bool { return consensus.Decider.IndexOf(pubKey) != -1 } diff --git a/consensus/validator.go b/consensus/validator.go index 7336c7b477..2ccc453645 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -408,7 +408,7 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { func (consensus *Consensus) getPriKeysInCommittee() []*bls.PrivateKeyWrapper { priKeys := []*bls.PrivateKeyWrapper{} for i, key := range consensus.priKey { - if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { + if !consensus.isValidatorInCommittee(key.Pub.Bytes) { continue } priKeys = append(priKeys, &consensus.priKey[i]) diff --git a/consensus/view_change.go b/consensus/view_change.go index 720205ee83..1a2accf727 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -279,7 +279,7 @@ func (consensus *Consensus) startViewChange() { // for view change, send separate view change per public key // do not do multi-sign of view change message for _, key := range consensus.priKey { - if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { + if !consensus.isValidatorInCommittee(key.Pub.Bytes) { continue } msgToSend := consensus.constructViewChangeMessage(&key) From d1a0e9cbf254b59d610dd52d93b31718ed25568a Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 7 Nov 2022 20:29:44 +0700 Subject: [PATCH 340/420] Fixed locks. --- consensus/consensus_service.go | 11 ++++++----- consensus/consensus_v2.go | 9 ++++++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index fdf0880625..daaa219842 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -74,6 +74,8 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa // UpdatePublicKeys updates the PublicKeys for // quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() consensus.Decider.UpdateParticipants(pubKeys, allowlist) consensus.getLogger().Info().Msg("My Committee updated") for i := range pubKeys { @@ -94,7 +96,7 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi } // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. - consensus.UpdateBitmaps() + consensus.updateBitmaps() consensus.resetState() // do not reset view change state if it is in view changing mode @@ -126,7 +128,7 @@ func (consensus *Consensus) signConsensusMessage(message *msg_pb.Message, } // UpdateBitmaps update the bitmaps for prepare and commit phase -func (consensus *Consensus) UpdateBitmaps() { +func (consensus *Consensus) updateBitmaps() { consensus.getLogger().Debug(). Str("MessageType", consensus.phase.String()). Msg("[UpdateBitmaps] Updating consensus bitmaps") @@ -440,9 +442,8 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { - // TODO: if remove locks blockchain stucks. - //consensus.mutex.RLock() - //defer consensus.mutex.RUnlock() + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() return consensus.isLeader() } diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 4253a46e95..37adeea430 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -703,11 +703,14 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess } consensus.FinishFinalityCount() - consensus.PostConsensusJob(blk) - consensus.SetupForNewConsensus(blk, committedMsg) + go func() { + consensus.PostConsensusJob(blk) + }() + consensus.setupForNewConsensus(blk, committedMsg) utils.Logger().Info().Uint64("blockNum", blk.NumberU64()). Str("hash", blk.Header().Hash().Hex()). Msg("Added New Block to Blockchain!!!") + return nil } @@ -756,7 +759,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { } // SetupForNewConsensus sets the state for new consensus -func (consensus *Consensus) SetupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { +func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) consensus.SetCurBlockViewID(committedMsg.ViewID + 1) consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] From 69ae68c7443a59f7fd83953397127b03a2ccaf52 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 8 Nov 2022 01:05:23 +0700 Subject: [PATCH 341/420] Fixed tests. --- consensus/consensus_service.go | 6 +++++- internal/utils/singleton.go | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index daaa219842..e54f5edf21 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -442,7 +442,11 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { - consensus.mutex.RLock() + _ = utils.AssertNoLongerThan0(5*time.Second, func() error { + consensus.mutex.RLock() + return nil + }) + defer consensus.mutex.RUnlock() return consensus.isLeader() diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index 1b71e981b5..9b94e8a35d 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path" + "runtime/debug" "strconv" "sync" "time" @@ -213,3 +214,19 @@ func GetPort() int { } return 0 } + +func AssertNoLongerThan0[E any](t time.Duration, f func() E) E { + ch := make(chan E) + defer close(ch) + stack := debug.Stack() + go func() { + select { + case <-time.After(t): + panic("AssertNoLongerThan0: " + string(stack)) + case <-ch: + return + } + }() + + return f() +} From 53dfc9d3572bb795eaa92466107dd01eff5e57d8 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 8 Nov 2022 13:34:53 +0700 Subject: [PATCH 342/420] Fixed lock. --- consensus/consensus_service.go | 37 +++++++++++++++++++++++++++++----- consensus/consensus_v2.go | 10 ++++----- internal/utils/singleton.go | 16 +++++++++++++++ 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e54f5edf21..e539264dbe 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -76,6 +76,10 @@ func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Messa func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { consensus.mutex.Lock() defer consensus.mutex.Unlock() + return consensus.updatePublicKeys(pubKeys, allowlist) +} + +func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { consensus.Decider.UpdateParticipants(pubKeys, allowlist) consensus.getLogger().Info().Msg("My Committee updated") for i := range pubKeys { @@ -172,6 +176,13 @@ func (consensus *Consensus) isValidatorInCommittee(pubKey bls.SerializedPublicKe // SetMode sets the mode of consensus func (consensus *Consensus) SetMode(m Mode) { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + consensus.setMode(m) +} + +// SetMode sets the mode of consensus +func (consensus *Consensus) setMode(m Mode) { if m == Normal && consensus.isBackup { m = NormalBackup } @@ -212,8 +223,8 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if consensus.IgnoreViewIDCheck.IsSet() { //in syncing mode, node accepts incoming messages without viewID/leaderKey checking //so only set mode to normal when new node enters consensus and need checking viewID - consensus.SetMode(Normal) - consensus.SetViewIDs(msg.ViewID) + consensus.setMode(Normal) + consensus.setViewIDs(msg.ViewID) if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } @@ -268,6 +279,12 @@ func (consensus *Consensus) readSignatureBitmapPayload(recvPayload []byte, offse // (b) node in committed but has any err during processing: Syncing mode // (c) node in committed and everything looks good: Normal mode func (consensus *Consensus) UpdateConsensusInformation() Mode { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + return consensus.updateConsensusInformation() +} + +func (consensus *Consensus) updateConsensusInformation() Mode { curHeader := consensus.Blockchain().CurrentHeader() curEpoch := curHeader.Epoch() nextEpoch := new(big.Int).Add(curHeader.Epoch(), common.Big1) @@ -372,7 +389,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { consensus.getLogger().Info(). Int("numPubKeys", len(pubKeys)). Msg("[UpdateConsensusInformation] Successfully updated public keys") - consensus.UpdatePublicKeys(pubKeys, shard.Schedule.InstanceForEpoch(nextEpoch).ExternalAllowlist()) + consensus.updatePublicKeys(pubKeys, shard.Schedule.InstanceForEpoch(nextEpoch).ExternalAllowlist()) // Update voters in the committee if _, err := consensus.Decider.SetVoters( @@ -485,8 +502,8 @@ func (consensus *Consensus) SetViewIDs(height uint64) { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) setViewIDs(height uint64) { - consensus.SetCurBlockViewID(height) - consensus.SetViewChangingID(height) + consensus.setCurBlockViewID(height) + consensus.setViewChangingID(height) } // SetCurBlockViewID set the current view ID @@ -494,11 +511,21 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } +// SetCurBlockViewID set the current view ID +func (consensus *Consensus) setCurBlockViewID(viewID uint64) { + consensus.current.SetCurBlockViewID(viewID) +} + // SetViewChangingID set the current view change ID func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) } +// SetViewChangingID set the current view change ID +func (consensus *Consensus) setViewChangingID(viewID uint64) { + consensus.current.SetViewChangingID(viewID) +} + // StartFinalityCount set the finality counter to current time func (consensus *Consensus) StartFinalityCount() { consensus.finalityCounter.Store(time.Now().UnixNano()) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 37adeea430..a30bc392a9 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -348,7 +348,7 @@ func (consensus *Consensus) syncReadyChan() { if consensus.BlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) consensus.SetViewIDs(consensus.Blockchain().CurrentHeader().ViewID().Uint64() + 1) - mode := consensus.UpdateConsensusInformation() + mode := consensus.updateConsensusInformation() consensus.current.SetMode(mode) consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") consensus.consensusTimeout[timeoutConsensus].Start() @@ -357,8 +357,8 @@ func (consensus *Consensus) syncReadyChan() { } else if consensus.Mode() == Syncing { // Corner case where sync is triggered before `onCommitted` and there is a race // for block insertion between consensus and downloader. - mode := consensus.UpdateConsensusInformation() - consensus.SetMode(mode) + mode := consensus.updateConsensusInformation() + consensus.setMode(mode) consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") consensus.consensusTimeout[timeoutConsensus].Start() consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() @@ -761,7 +761,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { // SetupForNewConsensus sets the state for new consensus func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg *FBFTMessage) { atomic.StoreUint64(&consensus.blockNum, blk.NumberU64()+1) - consensus.SetCurBlockViewID(committedMsg.ViewID + 1) + consensus.setCurBlockViewID(committedMsg.ViewID + 1) consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] var epoch *big.Int if blk.IsLastBlockInEpoch() { @@ -775,7 +775,7 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg // Update consensus keys at last so the change of leader status doesn't mess up normal flow if blk.IsLastBlockInEpoch() { - consensus.SetMode(consensus.UpdateConsensusInformation()) + consensus.setMode(consensus.updateConsensusInformation()) } consensus.FBFTLog.PruneCacheBeforeBlock(blk.NumberU64()) consensus.resetState() diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index 9b94e8a35d..f4e771c7f5 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -230,3 +230,19 @@ func AssertNoLongerThan0[E any](t time.Duration, f func() E) E { return f() } + +func AssertNoLongerThan(t time.Duration, f func()) { + ch := make(chan struct{}) + defer close(ch) + stack := debug.Stack() + go func() { + select { + case <-time.After(t): + panic("AssertNoLongerThan0: " + string(stack)) + case <-ch: + return + } + }() + + f() +} From 31a937474b9611ec70d96b10f3151ecb1bab887f Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 8 Nov 2022 19:30:57 +0700 Subject: [PATCH 343/420] Rebased over leader rotation. --- consensus/consensus_service.go | 18 +----------------- consensus/consensus_v2.go | 18 ++++-------------- consensus/view_change.go | 2 +- 3 files changed, 6 insertions(+), 32 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e539264dbe..1062b4502a 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -459,11 +459,7 @@ func (consensus *Consensus) updateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { - _ = utils.AssertNoLongerThan0(5*time.Second, func() error { - consensus.mutex.RLock() - return nil - }) - + consensus.mutex.RLock() defer consensus.mutex.RUnlock() return consensus.isLeader() @@ -481,18 +477,6 @@ func (consensus *Consensus) isLeader() bool { return false } -// isLeader check if the node is a leader or not by comparing the public key of -// the node with the leader public key. This function assume it runs under lock. -func (consensus *Consensus) isLeader() bool { - obj := consensus.LeaderPubKey.Object - for _, key := range consensus.priKey { - if key.Pub.Object.IsEqual(obj) { - return true - } - } - return false -} - // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index a30bc392a9..b8696200c3 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -460,16 +460,6 @@ func (consensus *Consensus) BlockChannel(newBlock *types.Block) { } } -// Close closes the consensus. If current is in normal commit phase, wait until the commit -// phase end. -func (consensus *Consensus) Close() error { - if consensus.dHelper != nil { - consensus.dHelper.close() - } - consensus.waitForCommit() - return nil -} - // waitForCommit wait extra 2 seconds for commit phase to finish func (consensus *Consensus) waitForCommit() { if consensus.Mode() != Normal || consensus.phase.Get() != FBFTCommit { @@ -717,11 +707,11 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess // rotateLeader rotates the leader to the next leader in the committee. // This function must be called with enabled leader rotation. func (consensus *Consensus) rotateLeader(epoch *big.Int) { - prev := consensus.GetLeaderPubKey() + prev := consensus.getLeaderPubKey() bc := consensus.Blockchain() curNumber := bc.CurrentHeader().Number().Uint64() utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), bc.Config().IsLeaderRotation(epoch), bc.Config().LeaderRotationBlocksCount) - leader := consensus.GetLeaderPubKey() + leader := consensus.getLeaderPubKey() for i := 0; i < bc.Config().LeaderRotationBlocksCount; i++ { header := bc.GetHeaderByNumber(curNumber - uint64(i)) if header == nil { @@ -748,9 +738,9 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { utils.Logger().Error().Msg("Failed to get next leader") return } else { - consensus.SetLeaderPubKey(next) + consensus.setLeaderPubKey(next) } - if consensus.IsLeader() && !consensus.GetLeaderPubKey().Object.IsEqual(prev.Object) { + if consensus.isLeader() && !consensus.getLeaderPubKey().Object.IsEqual(prev.Object) { // leader changed go func() { consensus.ReadySignal <- SyncProposal diff --git a/consensus/view_change.go b/consensus/view_change.go index 1a2accf727..d7080c5a79 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -340,7 +340,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.resetState() } - consensus.SetLeaderPubKey(newLeaderPriKey.Pub) + consensus.setLeaderPubKey(newLeaderPriKey.Pub) return nil } From 7dd6e46c8b330d72b78a3120803d7e0e54c6a3e0 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 1 Sep 2022 00:53:16 +0700 Subject: [PATCH 344/420] in progress. --- api/service/explorer/service.go | 8 +++++--- cmd/harmony/main.go | 3 +++ consensus/consensus.go | 2 ++ consensus/consensus_service.go | 29 +++++++++++++++++++++++++++++ consensus/consensus_v2.go | 17 ++++++++++------- consensus/leader.go | 1 + consensus/quorum/quorum.go | 7 ++++++- consensus/validator.go | 4 ++++ consensus/view_change.go | 8 ++++++++ internal/utils/singleton.go | 33 +-------------------------------- node/node_newblock.go | 2 -- 11 files changed, 69 insertions(+), 45 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 6aa275332b..d96c1bc56c 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -115,6 +115,8 @@ func (s *Service) Run() *http.Server { s.router = mux.NewRouter() + fmt.Println("++", addr) + // Set up router for addresses. // Fetch addresses request, accepts parameter size: how much addresses to read, // parameter prefix: from which address prefix start @@ -221,9 +223,9 @@ func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { for i := cur; i > 0; i-- { block := s.blockchain.GetBlockByNumber(i) - w.Write([]byte(fmt.Sprintf("#%d ", i))) - w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) - w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) + w.Write([]byte(fmt.Sprintf("%d ", i))) + w.Write([]byte(fmt.Sprintf("%s ", block.Header().ViewID().String()))) + w.Write([]byte(fmt.Sprintf("%s ", block.Header().Coinbase().Hash().Hex()))) w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) } } diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index f62b81acd8..0527a0b8c3 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -266,6 +266,8 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { var err error + fmt.Println("OS: ", os.Args) + nodeconfigSetShardSchedule(hc) nodeconfig.SetShardingSchedule(shard.Schedule) nodeconfig.SetVersion(getHarmonyVersion()) @@ -785,6 +787,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // Set the consensus ID to be the current block number viewID := currentNode.Blockchain().CurrentBlock().Header().ViewID().Uint64() + fmt.Println("viewID:", viewID) currentConsensus.SetViewIDs(viewID + 1) utils.Logger().Info(). Uint64("viewID", viewID). diff --git a/consensus/consensus.go b/consensus/consensus.go index a9fcc3cc52..8477eb2aa9 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -71,6 +71,8 @@ type Consensus struct { priKey multibls.PrivateKeys // the publickey of leader LeaderPubKey *bls.PublicKeyWrapper + // index of leader in the list of validators. + LeaderIndex int // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 1062b4502a..2fabb42b3c 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync/atomic" "time" @@ -228,6 +229,7 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } + fmt.Println("[checkViewID] Set LEADEER PUB KEY ", msg.SenderPubkeys[0].Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() @@ -477,9 +479,22 @@ func (consensus *Consensus) isLeader() bool { return false } +// isLeader check if the node is a leader or not by comparing the public key of +// the node with the leader public key. This function assume it runs under lock. +func (consensus *Consensus) isLeader() bool { + obj := consensus.LeaderPubKey.Object + for _, key := range consensus.priKey { + if key.Pub.Object.IsEqual(obj) { + return true + } + } + return false +} + // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { + fmt.Println("SetViewIDs", height) consensus.setViewIDs(height) } @@ -495,6 +510,19 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } +// SetLeaderIndex set the leader index. +func (consensus *Consensus) SetLeaderIndex(f func(int) int) (current int) { + consensus.pubKeyLock.Lock() + defer consensus.pubKeyLock.Unlock() + consensus.LeaderIndex = f(consensus.LeaderIndex) + return consensus.LeaderIndex +} + +func (consensus *Consensus) GetLeaderIndex() int { + consensus.pubKeyLock.Lock() + defer consensus.pubKeyLock.Unlock() + return consensus.LeaderIndex +} // SetCurBlockViewID set the current view ID func (consensus *Consensus) setCurBlockViewID(viewID uint64) { consensus.current.SetCurBlockViewID(viewID) @@ -505,6 +533,7 @@ func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) } + // SetViewChangingID set the current view change ID func (consensus *Consensus) setViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index b8696200c3..fea7b7dfe6 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -425,18 +425,20 @@ func (consensus *Consensus) Close() error { } func (consensus *Consensus) BlockChannel(newBlock *types.Block) { - //consensus.ReshardingNextLeader(newBlock)consensus.getLogger().Info(). - Uint64("MsgBlockNum", newBlock.NumberU64()). - Msg("[ConsensusMainLoop] Received Proposed New Block!") + //consensus.ReshardingNextLeader(newBlock) + consensus.getLogger().Info(). + Uint64("MsgBlockNum", newBlock.NumberU64()). + Msg("[ConsensusMainLoop] Received Proposed New Block!") if newBlock.NumberU64() < consensus.BlockNum() { consensus.getLogger().Warn().Uint64("newBlockNum", newBlock.NumberU64()). Msg("[ConsensusMainLoop] received old block, abort") return - } - // Sleep to wait for the full block time - consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") - time.AfterFunc(time.Until(consensus.NextBlockDue), func() { + } + // Sleep to wait for the full block time + consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") + + time.AfterFunc(time.Until(consensus.NextBlockDue), func() { consensus.StartFinalityCount() consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -664,6 +666,7 @@ func (consensus *Consensus) tryCatchup() error { consensus.getLogger().Error().Err(err).Msg("[TryCatchup] Failed to add block to chain") return err } + //fmt.Println("tryCatchup ", utils.GetPort(), blk.NumberU64()) select { // TODO: Remove this when removing dns sync and stream sync is fully up case consensus.VerifiedNewBlock <- blk: diff --git a/consensus/leader.go b/consensus/leader.go index 2f7766e19b..a1c7abb581 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -322,4 +322,5 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED) } + //fmt.Println("onCommit99: ", utils.GetPort(), recvMsg.BlockNum) } diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index da1551cdf9..6aa9b0dd79 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -229,12 +229,17 @@ func (s *cIdentities) NthNextHmy(instance shardingconfig.Instance, pubKey *bls.P Msg("[NthNextHmy] pubKey not found") } numNodes := instance.NumHarmonyOperatedNodesPerShard() + //fmt.Println("??idx:", idx, numNodes) // sanity check to avoid out of bound access if numNodes <= 0 || numNodes > len(s.publicKeys) { numNodes = len(s.publicKeys) } idx = (idx + next) % numNodes - return found, &s.publicKeys[idx] + //fmt.Println("-------idx:", idx) + + new := &s.publicKeys[idx] + fmt.Println("NthNextHmy: ", pubKey.Bytes.Hex(), new.Bytes.Hex()) + return found, new } // NthNextHmyExt return the Nth next pubkey of Harmony + allowlist nodes, next can be negative number diff --git a/consensus/validator.go b/consensus/validator.go index 2ccc453645..2b18003212 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -18,6 +18,7 @@ import ( ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { + recvMsg, err := consensus.parseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). @@ -26,6 +27,9 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Msg("[OnAnnounce] Unparseable leader message") return } + if consensus.ShardID == 0 { + //fmt.Println("onAnnounce called ", recvMsg.BlockNum) + } // NOTE let it handle its own logs if !consensus.onAnnounceSanityChecks(recvMsg) { diff --git a/consensus/view_change.go b/consensus/view_change.go index d7080c5a79..fb1b995f48 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "time" @@ -141,6 +142,7 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { Uint64("stuckBlockViewID", stuckBlockViewID). Msg("[getNextViewID]") + fmt.Println("end getNextViewID: ", nextViewID, viewChangeDuration) // duration is always the fixed view change duration for synchronous view change return nextViewID, viewChangeDuration } @@ -213,6 +215,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe lastLeaderPubKey, gap) } + fmt.Println("wasfoundNext", consensus.Blockchain.Config().IsAllowlistEpoch(epoch), wasFound, next.Bytes.Hex(), lastLeaderPubKey.Bytes.Hex()) if !wasFound { consensus.getLogger().Warn(). Str("key", consensus.LeaderPubKey.Bytes.Hex()). @@ -234,6 +237,7 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { + fmt.Printf("Message to send leader111: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } @@ -242,6 +246,7 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() + //fmt.Println("startViewChange", nextViewID) consensus.SetViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet @@ -282,7 +287,9 @@ func (consensus *Consensus) startViewChange() { if !consensus.isValidatorInCommittee(key.Pub.Bytes) { continue } + // Тут уже другой leader msgToSend := consensus.constructViewChangeMessage(&key) + fmt.Println("Message to send leader222: ", consensus.LeaderPubKey.Bytes.Hex()) if err := consensus.msgSender.SendWithRetry( consensus.BlockNum(), msg_pb.MessageType_VIEWCHANGE, @@ -340,6 +347,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.resetState() } + fmt.Println("[startNewView]", newLeaderPriKey.Pub.Bytes.Hex()) consensus.setLeaderPubKey(newLeaderPriKey.Pub) return nil diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index f4e771c7f5..6dee772fba 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -8,6 +8,7 @@ import ( "path" "runtime/debug" "strconv" + "strconv" "sync" "time" @@ -214,35 +215,3 @@ func GetPort() int { } return 0 } - -func AssertNoLongerThan0[E any](t time.Duration, f func() E) E { - ch := make(chan E) - defer close(ch) - stack := debug.Stack() - go func() { - select { - case <-time.After(t): - panic("AssertNoLongerThan0: " + string(stack)) - case <-ch: - return - } - }() - - return f() -} - -func AssertNoLongerThan(t time.Duration, f func()) { - ch := make(chan struct{}) - defer close(ch) - stack := debug.Stack() - go func() { - select { - case <-time.After(t): - panic("AssertNoLongerThan0: " + string(stack)) - case <-ch: - return - } - }() - - f() -} diff --git a/node/node_newblock.go b/node/node_newblock.go index c29fb1b96a..69523861f2 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -89,8 +89,6 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp newBlock, err := node.ProposeNewBlock(newCommitSigsChan) if err == nil { - fmt.Printf("ProposeNewBlock: #%d :%d e:%d, sh:%d with leader %s\n", newBlock.NumberU64(), utils.GetPort(), node.Blockchain().CurrentHeader().Epoch().Uint64(), node.Blockchain().ShardID(), node.Consensus.GetLeaderPubKey().Bytes.Hex()) - if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). Msg("Block with the same number was already proposed, abort.") From d1b3a5ba9f1e4814e103d61c3b6b86bbd2291809 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 5 Sep 2022 19:39:49 +0700 Subject: [PATCH 345/420] consensus check is forked --- api/service/explorer/service.go | 6 +++--- consensus/consensus.go | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index d96c1bc56c..5887bb3a02 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -223,9 +223,9 @@ func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { for i := cur; i > 0; i-- { block := s.blockchain.GetBlockByNumber(i) - w.Write([]byte(fmt.Sprintf("%d ", i))) - w.Write([]byte(fmt.Sprintf("%s ", block.Header().ViewID().String()))) - w.Write([]byte(fmt.Sprintf("%s ", block.Header().Coinbase().Hash().Hex()))) + w.Write([]byte(fmt.Sprintf("#%d ", i))) + w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) + w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) } } diff --git a/consensus/consensus.go b/consensus/consensus.go index 8477eb2aa9..a9fcc3cc52 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -71,8 +71,6 @@ type Consensus struct { priKey multibls.PrivateKeys // the publickey of leader LeaderPubKey *bls.PublicKeyWrapper - // index of leader in the list of validators. - LeaderIndex int // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 From 1b64682f1cf7f60eb09f8f066fa455c3d079bc17 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 5 Sep 2022 22:41:06 +0700 Subject: [PATCH 346/420] update master --- api/service/explorer/service.go | 2 ++ consensus/view_change.go | 2 +- node/node_newblock.go | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 5887bb3a02..1ebfa94ba1 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -223,6 +223,8 @@ func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { for i := cur; i > 0; i-- { block := s.blockchain.GetBlockByNumber(i) + leaderPubKey, _ := chain.GetLeaderPubKeyFromCoinbase(s.backend.Blockchain(), block.Header()) + w.Write([]byte(fmt.Sprintf("%s ", leaderPubKey.Bytes.Hex()))) w.Write([]byte(fmt.Sprintf("#%d ", i))) w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) diff --git a/consensus/view_change.go b/consensus/view_change.go index fb1b995f48..226db683c4 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -237,7 +237,7 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - fmt.Printf("Message to send leader111: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) + fmt.Printf("[startViewChange]: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } diff --git a/node/node_newblock.go b/node/node_newblock.go index 69523861f2..f96477413f 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -146,7 +146,6 @@ func (node *Node) ProposeNewBlock(commitSigs chan []byte) (*types.Block, error) if node.Blockchain().Config().IsStaking(header.Epoch()) { blsPubKeyBytes := leaderKey.Object.GetAddress() coinbase.SetBytes(blsPubKeyBytes[:]) - fmt.Println("coinbase.SetBytes leader: ", leaderKey.Bytes.Hex(), coinbase.Hex()) } emptyAddr := common.Address{} From f907c70920a7093a63c3ce8e52ad76b62b4347d7 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 6 Sep 2022 02:58:48 +0700 Subject: [PATCH 347/420] fix leader --- consensus/consensus.go | 1 + consensus/consensus_service.go | 34 ++++++++++++++++++++++++++-------- consensus/consensus_v2.go | 2 +- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index a9fcc3cc52..9ab844e734 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -74,6 +74,7 @@ type Consensus struct { // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 + epoch uint64 // Blockhash - 32 byte blockHash [32]byte // Block to run consensus on diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 2fabb42b3c..76e5d9c44b 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -82,6 +82,7 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { consensus.Decider.UpdateParticipants(pubKeys, allowlist) + consensus.pubKeyLock.Unlock() consensus.getLogger().Info().Msg("My Committee updated") for i := range pubKeys { consensus.getLogger().Info(). @@ -89,15 +90,22 @@ func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.Publi Str("BLSPubKey", pubKeys[i].Bytes.Hex()). Msg("Member") } - - allKeys := consensus.Decider.Participants() - if len(allKeys) != 0 { - consensus.LeaderPubKey = &allKeys[0] - consensus.getLogger().Info(). - Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") + if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { + consensus.updateLeader() } else { - consensus.getLogger().Error(). - Msg("[UpdatePublicKeys] Participants is empty") + consensus.pubKeyLock.Lock() + allKeys := consensus.Decider.Participants() + consensus.pubKeyLock.Unlock() + if len(allKeys) != 0 { + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = &allKeys[0] + consensus.pubKeyLock.Unlock() + consensus.getLogger().Info(). + Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") + } else { + consensus.getLogger().Error(). + Msg("[UpdatePublicKeys] Participants is empty") + } } // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. @@ -658,3 +666,13 @@ func (consensus *Consensus) getLogger() *zerolog.Logger { Logger() return &logger } + +func UpdatePublicKeyDefault(consensus *Consensus) { + if allKeys := consensus.Decider.Participants(); len(allKeys) > 0 { + consensus.LeaderPubKey = &allKeys[0] + } +} + +func UpdatePublicKeyRotate(consensus *Consensus) { + //consensus +} diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index fea7b7dfe6..653ae72d66 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -778,7 +778,7 @@ func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, er if epoch.Uint64() == 0 { return 0, nil } - epochBlock := consensus.Blockchain.GetBlockByNumber(epoch.Uint64() - 1) + epochBlock := consensus.Blockchain.GetBlockByNumber(shard.Schedule.EpochLastBlock(epoch.Uint64() - 1)) if epochBlock == nil { return 0, errors.Errorf("block not found for number %d", epoch.Uint64()-1) } From 6d5753ccdd7c92de2458ce7a7c3073cc0580d537 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 7 Sep 2022 22:20:05 +0700 Subject: [PATCH 348/420] check leader for N blocks --- consensus/consensus.go | 5 +++++ consensus/consensus_service.go | 31 +++++++++++++------------------ consensus/consensus_v2.go | 4 +--- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 9ab844e734..75c83640bf 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -187,6 +187,11 @@ func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper { func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } +func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = pub + consensus.pubKeyLock.Unlock() +} func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 76e5d9c44b..5e553f2628 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -82,31 +82,26 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { consensus.Decider.UpdateParticipants(pubKeys, allowlist) + allKeys := consensus.Decider.Participants() consensus.pubKeyLock.Unlock() - consensus.getLogger().Info().Msg("My Committee updated") + if len(allKeys) != 0 { + first := consensus.Decider.FirstParticipant( + shard.Schedule.InstanceForEpoch(consensus.Blockchain.CurrentHeader().Epoch())) + consensus.pubKeyLock.Lock() + consensus.LeaderPubKey = first + consensus.pubKeyLock.Unlock() + consensus.getLogger().Info(). + Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") + } else { + consensus.getLogger().Error(). + Msg("[UpdatePublicKeys] Participants is empty") + } for i := range pubKeys { consensus.getLogger().Info(). Int("index", i). Str("BLSPubKey", pubKeys[i].Bytes.Hex()). Msg("Member") } - if consensus.Blockchain.Config().IsLeaderRotation(consensus.GetCurEpoch()) { - consensus.updateLeader() - } else { - consensus.pubKeyLock.Lock() - allKeys := consensus.Decider.Participants() - consensus.pubKeyLock.Unlock() - if len(allKeys) != 0 { - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = &allKeys[0] - consensus.pubKeyLock.Unlock() - consensus.getLogger().Info(). - Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") - } else { - consensus.getLogger().Error(). - Msg("[UpdatePublicKeys] Participants is empty") - } - } // reset states after update public keys // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. consensus.updateBitmaps() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 653ae72d66..8a3d4110b2 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/hex" - "fmt" "math/big" "sync/atomic" "time" @@ -12,6 +11,7 @@ import ( "github.com/ethereum/go-ethereum/common" bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" + "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" @@ -437,7 +437,6 @@ func (consensus *Consensus) BlockChannel(newBlock *types.Block) { } // Sleep to wait for the full block time consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") - time.AfterFunc(time.Until(consensus.NextBlockDue), func() { consensus.StartFinalityCount() consensus.mutex.Lock() @@ -548,7 +547,6 @@ func (consensus *Consensus) getLastMileBlocksAndMsg(bnStart uint64) ([]*types.Bl // preCommitAndPropose commit the current block with 67% commit signatures and start // proposing new block which will wait on the full commit signatures to finish func (consensus *Consensus) preCommitAndPropose(blk *types.Block) error { - //fmt.Println("preCommitAndPropose", utils.GetPort(), blk.NumberU64()) if blk == nil { return errors.New("block to pre-commit is nil") } From 411ec43ef1557c56b2c085465a08cfe395e2b6ae Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sat, 10 Sep 2022 18:02:45 +0700 Subject: [PATCH 349/420] fix --- api/service/explorer/service.go | 2 -- cmd/harmony/main.go | 3 --- consensus/consensus_service.go | 11 ----------- consensus/quorum/quorum.go | 7 +------ 4 files changed, 1 insertion(+), 22 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 1ebfa94ba1..b13f73db61 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -115,8 +115,6 @@ func (s *Service) Run() *http.Server { s.router = mux.NewRouter() - fmt.Println("++", addr) - // Set up router for addresses. // Fetch addresses request, accepts parameter size: how much addresses to read, // parameter prefix: from which address prefix start diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 0527a0b8c3..f62b81acd8 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -266,8 +266,6 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { var err error - fmt.Println("OS: ", os.Args) - nodeconfigSetShardSchedule(hc) nodeconfig.SetShardingSchedule(shard.Schedule) nodeconfig.SetVersion(getHarmonyVersion()) @@ -787,7 +785,6 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // Set the consensus ID to be the current block number viewID := currentNode.Blockchain().CurrentBlock().Header().ViewID().Uint64() - fmt.Println("viewID:", viewID) currentConsensus.SetViewIDs(viewID + 1) utils.Logger().Info(). Uint64("viewID", viewID). diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 5e553f2628..3f75c0bb00 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -232,7 +232,6 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { if !msg.HasSingleSender() { return errors.New("Leader message can not have multiple sender keys") } - fmt.Println("[checkViewID] Set LEADEER PUB KEY ", msg.SenderPubkeys[0].Bytes.Hex(), utils.GetPort()) consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() @@ -661,13 +660,3 @@ func (consensus *Consensus) getLogger() *zerolog.Logger { Logger() return &logger } - -func UpdatePublicKeyDefault(consensus *Consensus) { - if allKeys := consensus.Decider.Participants(); len(allKeys) > 0 { - consensus.LeaderPubKey = &allKeys[0] - } -} - -func UpdatePublicKeyRotate(consensus *Consensus) { - //consensus -} diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 6aa9b0dd79..da1551cdf9 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -229,17 +229,12 @@ func (s *cIdentities) NthNextHmy(instance shardingconfig.Instance, pubKey *bls.P Msg("[NthNextHmy] pubKey not found") } numNodes := instance.NumHarmonyOperatedNodesPerShard() - //fmt.Println("??idx:", idx, numNodes) // sanity check to avoid out of bound access if numNodes <= 0 || numNodes > len(s.publicKeys) { numNodes = len(s.publicKeys) } idx = (idx + next) % numNodes - //fmt.Println("-------idx:", idx) - - new := &s.publicKeys[idx] - fmt.Println("NthNextHmy: ", pubKey.Bytes.Hex(), new.Bytes.Hex()) - return found, new + return found, &s.publicKeys[idx] } // NthNextHmyExt return the Nth next pubkey of Harmony + allowlist nodes, next can be negative number From e1c0e0e9bd8af59edea64fd83200e30f40bc8d26 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 14:04:09 +0700 Subject: [PATCH 350/420] fix --- consensus/consensus_service.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 3f75c0bb00..d436ef5d56 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync/atomic" "time" @@ -496,7 +495,6 @@ func (consensus *Consensus) isLeader() bool { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { - fmt.Println("SetViewIDs", height) consensus.setViewIDs(height) } From d8b64c9af7c7bf01c2b2030ecd34a9069cb1f9a9 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 15:27:56 +0700 Subject: [PATCH 351/420] Cleanup and fix update pub keys. --- consensus/consensus_service.go | 16 ++++++++++------ consensus/consensus_v2.go | 1 - consensus/leader.go | 1 - consensus/view_change.go | 7 ------- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index d436ef5d56..1913460eb9 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "math/big" "sync/atomic" "time" @@ -81,14 +82,17 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys, allowlist []bls_cosi.Publi func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.PublicKeyWrapper) int64 { consensus.Decider.UpdateParticipants(pubKeys, allowlist) + consensus.getLogger().Info().Msg("My Committee updated") + for i := range pubKeys { + consensus.getLogger().Info(). + Int("index", i). + Str("BLSPubKey", pubKeys[i].Bytes.Hex()). + Msg("Member") + } + allKeys := consensus.Decider.Participants() - consensus.pubKeyLock.Unlock() if len(allKeys) != 0 { - first := consensus.Decider.FirstParticipant( - shard.Schedule.InstanceForEpoch(consensus.Blockchain.CurrentHeader().Epoch())) - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = first - consensus.pubKeyLock.Unlock() + consensus.LeaderPubKey = &allKeys[0] consensus.getLogger().Info(). Str("info", consensus.LeaderPubKey.Bytes.Hex()).Msg("My Leader") } else { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 8a3d4110b2..3ec460ef52 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -664,7 +664,6 @@ func (consensus *Consensus) tryCatchup() error { consensus.getLogger().Error().Err(err).Msg("[TryCatchup] Failed to add block to chain") return err } - //fmt.Println("tryCatchup ", utils.GetPort(), blk.NumberU64()) select { // TODO: Remove this when removing dns sync and stream sync is fully up case consensus.VerifiedNewBlock <- blk: diff --git a/consensus/leader.go b/consensus/leader.go index a1c7abb581..2f7766e19b 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -322,5 +322,4 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED) } - //fmt.Println("onCommit99: ", utils.GetPort(), recvMsg.BlockNum) } diff --git a/consensus/view_change.go b/consensus/view_change.go index 226db683c4..377a7098f7 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "time" @@ -142,7 +141,6 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { Uint64("stuckBlockViewID", stuckBlockViewID). Msg("[getNextViewID]") - fmt.Println("end getNextViewID: ", nextViewID, viewChangeDuration) // duration is always the fixed view change duration for synchronous view change return nextViewID, viewChangeDuration } @@ -215,7 +213,6 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe lastLeaderPubKey, gap) } - fmt.Println("wasfoundNext", consensus.Blockchain.Config().IsAllowlistEpoch(epoch), wasFound, next.Bytes.Hex(), lastLeaderPubKey.Bytes.Hex()) if !wasFound { consensus.getLogger().Warn(). Str("key", consensus.LeaderPubKey.Bytes.Hex()). @@ -237,7 +234,6 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - fmt.Printf("[startViewChange]: %d %s \n", utils.GetPort(), consensus.LeaderPubKey.Bytes.Hex()) if consensus.disableViewChange || consensus.IsBackup() { return } @@ -246,7 +242,6 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() - //fmt.Println("startViewChange", nextViewID) consensus.SetViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet @@ -287,9 +282,7 @@ func (consensus *Consensus) startViewChange() { if !consensus.isValidatorInCommittee(key.Pub.Bytes) { continue } - // Тут уже другой leader msgToSend := consensus.constructViewChangeMessage(&key) - fmt.Println("Message to send leader222: ", consensus.LeaderPubKey.Bytes.Hex()) if err := consensus.msgSender.SendWithRetry( consensus.BlockNum(), msg_pb.MessageType_VIEWCHANGE, From f0b72bed341c6381decf8b4e52fe7b38892aa893 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 12 Sep 2022 16:32:36 +0700 Subject: [PATCH 352/420] Rotate leader. --- consensus/consensus.go | 1 - 1 file changed, 1 deletion(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 75c83640bf..a6296406de 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -74,7 +74,6 @@ type Consensus struct { // blockNum: the next blockNumber that FBFT is going to agree on, // should be equal to the blockNumber of next block blockNum uint64 - epoch uint64 // Blockhash - 32 byte blockHash [32]byte // Block to run consensus on From 785b1ff9e8c192bad01c1e09073283adb058b427 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 13 Sep 2022 01:10:13 +0700 Subject: [PATCH 353/420] fix fix fix fix fix --- consensus/consensus_service.go | 1 - consensus/consensus_v2.go | 6 ++++++ internal/params/config.go | 11 +++++++---- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 1913460eb9..411b9f6552 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -1,7 +1,6 @@ package consensus import ( - "fmt" "math/big" "sync/atomic" "time" diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 3ec460ef52..e8fa0fb17c 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -759,6 +759,12 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg } else { epoch = blk.Epoch() } + var epoch *big.Int + if blk.IsLastBlockInEpoch() { + epoch = new(big.Int).Add(blk.Epoch(), common.Big1) + } else { + epoch = blk.Epoch() + } if consensus.Blockchain.Config().IsLeaderRotation(epoch) { consensus.rotateLeader(epoch) } diff --git a/internal/params/config.go b/internal/params/config.go index fceca81627..82a2b4032a 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -275,6 +275,8 @@ var ( LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 5, FeeCollectEpoch: big.NewInt(5), + LeaderRotationEpoch: big.NewInt(4), + LeaderRotationBlocksCount: 5, } // AllProtocolChanges ... @@ -314,8 +316,8 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } @@ -356,8 +358,9 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount + + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } From 7f5d4656f819d5622bc592d0c422bc9ddd3679cb Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 13 Sep 2022 02:15:25 +0700 Subject: [PATCH 354/420] Cleaned. --- api/service/explorer/service.go | 48 ------------------------------- cmd/harmony/main.go | 3 +- consensus/consensus_v2.go | 13 --------- consensus/validator.go | 11 ------- consensus/view_change.go | 7 ----- node/node_newblock.go | 1 - test/configs/local-resharding.txt | 31 ++++++++++++++------ test/deploy.sh | 5 +--- 8 files changed, 24 insertions(+), 95 deletions(-) diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index b13f73db61..b5ffa1b8b0 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -7,13 +7,11 @@ import ( "fmt" "net" "net/http" - "os" "path" "strconv" "time" "github.com/RoaringBitmap/roaring/roaring64" - "github.com/harmony-one/harmony/internal/common" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" ethCommon "github.com/ethereum/go-ethereum/common" @@ -121,9 +119,6 @@ func (s *Service) Run() *http.Server { s.router.Path("/addresses").Queries("size", "{[0-9]*?}", "prefix", "{[a-zA-Z0-9]*?}").HandlerFunc(s.GetAddresses).Methods("GET") s.router.Path("/addresses").HandlerFunc(s.GetAddresses) s.router.Path("/height").HandlerFunc(s.GetHeight) - s.router.Path("/leader").HandlerFunc(s.GetLeader) - s.router.Path("/blocks").HandlerFunc(s.GetBlocks) - s.router.Path("/halt").HandlerFunc(s.halt) // Set up router for supply info s.router.Path("/burn-addresses").Queries().HandlerFunc(s.GetInaccessibleAddressInfo).Methods("GET") @@ -191,49 +186,6 @@ type HeightResponse struct { S3 uint64 `json:"3,omitempty"` } -func (s *Service) GetLeader(w http.ResponseWriter, r *http.Request) { - if s.backend.IsCurrentlyLeader() { - w.Write([]byte("true ")) - } else { - w.Write([]byte("false")) - } - - keys := "" - for _, p := range s.backend.GetPublicKeys() { - addr := common.Address{} - addrBytes := p.Object.GetAddress() - addr.SetBytes(addrBytes[:]) - keys += fmt.Sprintf("%s ", addr.String()) - break - } - //blsPubKeyBytes := leaderKey.Object.GetAddress() - //coinbase.SetBytes(blsPubKeyBytes[:]) - - w.Write([]byte(fmt.Sprintf(" %d", s.blockchain.ShardID()))) - w.Write([]byte(fmt.Sprintf(" %s", s.Port))) - w.Write([]byte(fmt.Sprintf(" %s", keys))) - w.Write([]byte(fmt.Sprintf(" %s", s.backend.GetPublicKeys().SerializeToHexStr()))) - -} - -func (s *Service) GetBlocks(w http.ResponseWriter, r *http.Request) { - cur := s.blockchain.CurrentHeader().Number().Uint64() - - for i := cur; i > 0; i-- { - block := s.blockchain.GetBlockByNumber(i) - leaderPubKey, _ := chain.GetLeaderPubKeyFromCoinbase(s.backend.Blockchain(), block.Header()) - w.Write([]byte(fmt.Sprintf("%s ", leaderPubKey.Bytes.Hex()))) - w.Write([]byte(fmt.Sprintf("#%d ", i))) - w.Write([]byte(fmt.Sprintf("v%s ", block.Header().ViewID().String()))) - w.Write([]byte(fmt.Sprintf("e%d ", block.Header().Epoch().Uint64()))) - w.Write([]byte(fmt.Sprintf("%s\n", block.Header().Coinbase().Hex()))) - } -} - -func (s *Service) halt(w http.ResponseWriter, r *http.Request) { - os.Exit(0) -} - // GetHeight returns heights of current and beacon chains if needed. func (s *Service) GetHeight(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index f62b81acd8..f01cb758ed 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -409,9 +409,8 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { if currentNode.NodeConfig.Role() == nodeconfig.Validator { currentNode.RegisterValidatorServices() } else if currentNode.NodeConfig.Role() == nodeconfig.ExplorerNode { - + currentNode.RegisterExplorerServices() } - currentNode.RegisterExplorerServices() currentNode.RegisterService(service.CrosslinkSending, crosslink_sending.New(currentNode, currentNode.Blockchain())) if hc.Pprof.Enabled { setupPprofService(currentNode, hc) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index e8fa0fb17c..19d0c88907 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -678,8 +678,6 @@ func (consensus *Consensus) tryCatchup() error { } func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMessage) error { - // this function evaluates for all, leader and validators. - if consensus.Blockchain().CurrentBlock().NumberU64() < blk.NumberU64() { if _, err := consensus.Blockchain().InsertChain([]*types.Block{blk}, !consensus.FBFTLog.IsBlockVerified(blk.Hash())); err != nil { consensus.getLogger().Error().Err(err).Msg("[commitBlock] Failed to add block to chain") @@ -777,17 +775,6 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg consensus.resetState() } -func (consensus *Consensus) getEpochFirstBlockViewID(epoch *big.Int) (uint64, error) { - if epoch.Uint64() == 0 { - return 0, nil - } - epochBlock := consensus.Blockchain.GetBlockByNumber(shard.Schedule.EpochLastBlock(epoch.Uint64() - 1)) - if epochBlock == nil { - return 0, errors.Errorf("block not found for number %d", epoch.Uint64()-1) - } - return epochBlock.Header().ViewID().Uint64() + 1, nil -} - func (consensus *Consensus) postCatchup(initBN uint64) { if initBN < consensus.BlockNum() { consensus.getLogger().Info(). diff --git a/consensus/validator.go b/consensus/validator.go index 2b18003212..06aa9bc08c 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -18,7 +18,6 @@ import ( ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { - recvMsg, err := consensus.parseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). @@ -27,9 +26,6 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { Msg("[OnAnnounce] Unparseable leader message") return } - if consensus.ShardID == 0 { - //fmt.Println("onAnnounce called ", recvMsg.BlockNum) - } // NOTE let it handle its own logs if !consensus.onAnnounceSanityChecks(recvMsg) { @@ -398,13 +394,6 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { consensus.getLogger().Info().Msg("[OnCommitted] Start consensus timer (new block added)") consensus.consensusTimeout[timeoutConsensus].Start() } - - //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum) - if blk != nil { - //consensus.ReshardingNextLeader(blk) - } else { - //fmt.Println("onCommitted", utils.GetPort(), recvMsg.BlockNum, "blk is nil") - } } // Collect private keys that are part of the current committee. diff --git a/consensus/view_change.go b/consensus/view_change.go index 377a7098f7..06640c123d 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -367,13 +367,6 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { return } - consensus.getLogger().Debug(). - Err(err). - Interface("SenderPubkeys", recvMsg.SenderPubkeys). - Str("NextLeader", recvMsg.LeaderPubkey.Bytes.Hex()). - Str("myBLSPubKey", consensus.priKey.GetPublicKeys().SerializeToHexStr()). - Msg("[onViewChange] I am the Leader") - if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) { consensus.getLogger().Info(). Int64("have", consensus.Decider.SignersCount(quorum.ViewChange)). diff --git a/node/node_newblock.go b/node/node_newblock.go index f96477413f..5050e4d6a7 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -2,7 +2,6 @@ package node import ( "errors" - "fmt" "sort" "strings" "time" diff --git a/test/configs/local-resharding.txt b/test/configs/local-resharding.txt index fc3d3e97b4..4ce91ece1c 100644 --- a/test/configs/local-resharding.txt +++ b/test/configs/local-resharding.txt @@ -1,12 +1,25 @@ 127.0.0.1 9000 validator .hmy/65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204.key -127.0.0.1 9002 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key -127.0.0.1 9004 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key -127.0.0.1 9006 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key -127.0.0.1 9008 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key -127.0.0.1 9010 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key -127.0.0.1 9099 explorer null 0 +127.0.0.1 9002 validator .hmy/40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809.key +127.0.0.1 9004 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key +127.0.0.1 9006 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key +127.0.0.1 9008 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key +127.0.0.1 9010 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key +127.0.0.1 9012 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key +127.0.0.1 9014 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key +127.0.0.1 9016 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key +127.0.0.1 9018 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key +127.0.0.1 9020 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key +127.0.0.1 9022 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key +127.0.0.1 9200 explorer null 0 127.0.0.1 9100 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key -127.0.0.1 9102 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key -127.0.0.1 9104 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key -127.0.0.1 9106 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key \ No newline at end of file +127.0.0.1 9102 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key +127.0.0.1 9104 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key +127.0.0.1 9106 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key +127.0.0.1 9108 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key +127.0.0.1 9110 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key +127.0.0.1 9112 validator .hmy/eca09c1808b729ca56f1b5a6a287c6e1c3ae09e29ccf7efa35453471fcab07d9f73cee249e2b91f5ee44eb9618be3904.key +127.0.0.1 9114 validator .hmy/f47238daef97d60deedbde5302d05dea5de67608f11f406576e363661f7dcbc4a1385948549b31a6c70f6fde8a391486.key +127.0.0.1 9116 validator .hmy/fc4b9c535ee91f015efff3f32fbb9d32cdd9bfc8a837bb3eee89b8fff653c7af2050a4e147ebe5c7233dc2d5df06ee0a.key +127.0.0.1 9118 validator .hmy/ca86e551ee42adaaa6477322d7db869d3e203c00d7b86c82ebee629ad79cb6d57b8f3db28336778ec2180e56a8e07296.key +127.0.0.1 9300 explorer null 1 \ No newline at end of file diff --git a/test/deploy.sh b/test/deploy.sh index fe22de57f0..9ab6943110 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -69,7 +69,7 @@ function launch_localnet() { if ${VERBOSE}; then verbosity=5 else - verbosity=5 + verbosity=3 fi base_args=(--log_folder "${log_folder}" --min_peers "${MIN}" --bootnodes "${BN_MA}" "--network_type=$NETWORK" --blspass file:"${ROOT}/.hmy/blspass.txt" "--dns=false" "--verbosity=${verbosity}" "--p2p.security.max-conn-per-ip=100") @@ -80,11 +80,8 @@ function launch_localnet() { while IFS='' read -r line || [[ -n "$line" ]]; do i=$((i + 1)) - - # Read config for i-th node form config file IFS=' ' read -r ip port mode bls_key shard node_config <<<"${line}" - echo "LINE: ${line} ${shard}" args=("${base_args[@]}" --ip "${ip}" --port "${port}" --key "/tmp/${ip}-${port}.key" --db_dir "${ROOT}/db-${ip}-${port}" "--broadcast_invalid_tx=false") if [[ -z "$ip" || -z "$port" ]]; then echo "skip empty node" From a7352569f24f1848f175e92175ed1643c3bae5b4 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Mon, 19 Sep 2022 21:40:21 +0700 Subject: [PATCH 355/420] Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. --- consensus/consensus_v2.go | 1 - 1 file changed, 1 deletion(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 19d0c88907..1fcb6e2e4b 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -11,7 +11,6 @@ import ( "github.com/ethereum/go-ethereum/common" bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/signature" - "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" From 287b46cf2e3e72ed2536b2fbf0fbc227fa9780a6 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 8 Nov 2022 20:19:37 +0700 Subject: [PATCH 356/420] Fix failed tests. --- internal/params/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/params/config.go b/internal/params/config.go index 82a2b4032a..edd17feb45 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -316,8 +316,8 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch } @@ -361,7 +361,7 @@ var ( big.NewInt(1), // LeaderRotationEpoch 64, // LeaderRotationBlocksCount - big.NewInt(0), // FeeCollectEpoch + big.NewInt(0), // FeeCollectEpoch } // TestRules ... From 96ce60d58850a7a533470b6bc17688866f07be75 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 9 Nov 2022 15:36:28 +0700 Subject: [PATCH 357/420] Fixed code review. --- consensus/consensus_v2.go | 3 +-- hmy/hmy.go | 2 -- internal/params/config.go | 9 ++++----- node/api.go | 6 ------ 4 files changed, 5 insertions(+), 15 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 1fcb6e2e4b..e6e6445914 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -137,7 +137,6 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb } func (consensus *Consensus) finalCommit() { - // THIS IS NOT GOOD PLACE FOR LEADER SWITCHING numCommits := consensus.Decider.SignersCount(quorum.Commit) consensus.getLogger().Info(). @@ -714,7 +713,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { if header == nil { return } - // Previous epoch, we should not change leader. + // Previous block was epoch block, we should not change leader. if header.Epoch().Uint64() != epoch.Uint64() { return } diff --git a/hmy/hmy.go b/hmy/hmy.go index 8089f04ba8..58e8d59edf 100644 --- a/hmy/hmy.go +++ b/hmy/hmy.go @@ -18,7 +18,6 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" - "github.com/harmony-one/harmony/multibls" commonRPC "github.com/harmony-one/harmony/rpc/common" "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" @@ -97,7 +96,6 @@ type NodeAPI interface { GetStakingTransactionsCount(address, txType string) (uint64, error) GetTraceResultByHash(hash common.Hash) (json.RawMessage, error) IsCurrentlyLeader() bool - GetPublicKeys() multibls.PublicKeys IsOutOfSync(shardID uint32) bool SyncStatus(shardID uint32) (bool, uint64, uint64) SyncPeers() map[string]int diff --git a/internal/params/config.go b/internal/params/config.go index edd17feb45..af15107bcb 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -275,7 +275,7 @@ var ( LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 5, FeeCollectEpoch: big.NewInt(5), - LeaderRotationEpoch: big.NewInt(4), + LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 5, } @@ -358,10 +358,9 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount - big.NewInt(0), // FeeCollectEpoch + big.NewInt(1), // LeaderRotationEpoch + 64, // LeaderRotationBlocksCount + big.NewInt(0), // FeeCollectEpoch } // TestRules ... diff --git a/node/api.go b/node/api.go index debcb201f6..ceda968084 100644 --- a/node/api.go +++ b/node/api.go @@ -6,7 +6,6 @@ import ( "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy" "github.com/harmony-one/harmony/internal/tikv" - "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/rosetta" hmy_rpc "github.com/harmony-one/harmony/rpc" rpc_common "github.com/harmony-one/harmony/rpc/common" @@ -19,11 +18,6 @@ func (node *Node) IsCurrentlyLeader() bool { return node.Consensus.IsLeader() } -// GetPublicKeys exposes if node is currently the leader node -func (node *Node) GetPublicKeys() multibls.PublicKeys { - return node.Consensus.GetPrivateKeys().GetPublicKeys() -} - // PeerConnectivity .. func (node *Node) PeerConnectivity() (int, int, int) { return node.host.PeerConnectivity() From 731e17f19463305e396d290a83954e27407d77de Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Thu, 10 Nov 2022 12:57:57 +0700 Subject: [PATCH 358/420] Fix review comments. --- consensus/consensus.go | 5 +++++ consensus/consensus_v2.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index a6296406de..4f40050a3a 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -186,6 +186,11 @@ func (consensus *Consensus) GetLeaderPubKey() *bls_cosi.PublicKeyWrapper { func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } + +func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { + return consensus.LeaderPubKey +} + func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { consensus.pubKeyLock.Lock() consensus.LeaderPubKey = pub diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index e6e6445914..e9cdfe5ef4 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -713,7 +713,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { if header == nil { return } - // Previous block was epoch block, we should not change leader. + // Previous epoch, we should not change leader. if header.Epoch().Uint64() != epoch.Uint64() { return } From 661b937fd7b93bff679de153ba12ee2c38db0596 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 17 Jan 2023 13:22:30 -0300 Subject: [PATCH 359/420] Merged leader rotation. --- consensus/consensus_service.go | 12 ------------ consensus/consensus_v2.go | 15 +++++---------- internal/params/config.go | 2 -- internal/utils/singleton.go | 2 -- 4 files changed, 5 insertions(+), 26 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 411b9f6552..e62b2ed048 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -471,18 +471,6 @@ func (consensus *Consensus) IsLeader() bool { return consensus.isLeader() } -// IsLeader check if the node is a leader or not by comparing the public key of -// the node with the leader public key -func (consensus *Consensus) isLeader() bool { - obj := consensus.LeaderPubKey.Object - for _, key := range consensus.priKey { - if key.Pub.Object.IsEqual(obj) { - return true - } - } - return false -} - // isLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key. This function assume it runs under lock. func (consensus *Consensus) isLeader() bool { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index e9cdfe5ef4..3ef0ea0655 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -412,16 +412,6 @@ func (consensus *Consensus) tick() { } } -// Close closes the consensus. If current is in normal commit phase, wait until the commit -// phase end. -func (consensus *Consensus) Close() error { - if consensus.dHelper != nil { - consensus.dHelper.close() - } - consensus.waitForCommit() - return nil -} - func (consensus *Consensus) BlockChannel(newBlock *types.Block) { //consensus.ReshardingNextLeader(newBlock) consensus.getLogger().Info(). @@ -489,6 +479,11 @@ func (consensus *Consensus) GetLastMileBlockIter(bnStart uint64, cb func(iter *L consensus.mutex.Lock() defer consensus.mutex.Unlock() + return consensus.getLastMileBlockIter(bnStart, cb) +} + +// GetLastMileBlockIter get the iterator of the last mile blocks starting from number bnStart +func (consensus *Consensus) getLastMileBlockIter(bnStart uint64, cb func(iter *LastMileBlockIter) error) error { if consensus.BlockVerifier == nil { return errors.New("consensus haven't initialized yet") } diff --git a/internal/params/config.go b/internal/params/config.go index af15107bcb..fceca81627 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -275,8 +275,6 @@ var ( LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 5, FeeCollectEpoch: big.NewInt(5), - LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 5, } // AllProtocolChanges ... diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index 6dee772fba..1b71e981b5 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -6,8 +6,6 @@ import ( "fmt" "os" "path" - "runtime/debug" - "strconv" "strconv" "sync" "time" From 900be09cfa4ca27ffa8b9fb4a9ac8563c7fdb845 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sat, 21 Jan 2023 17:07:40 -0300 Subject: [PATCH 360/420] Rebased on dev. --- consensus/consensus.go | 6 ------ consensus/consensus_service.go | 16 +--------------- consensus/consensus_v2.go | 20 +++++++------------- consensus/view_change.go | 14 ++++++++++---- 4 files changed, 18 insertions(+), 38 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 4f40050a3a..6d396c100c 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -191,12 +191,6 @@ func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } -func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = pub - consensus.pubKeyLock.Unlock() -} - func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e62b2ed048..a92a71f7e4 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -111,7 +111,7 @@ func (consensus *Consensus) updatePublicKeys(pubKeys, allowlist []bls_cosi.Publi // do not reset view change state if it is in view changing mode if !consensus.isViewChangingMode() { - consensus.ResetViewChangeState() + consensus.resetViewChangeState() } return consensus.Decider.ParticipantsCount() } @@ -501,19 +501,6 @@ func (consensus *Consensus) SetCurBlockViewID(viewID uint64) uint64 { return consensus.current.SetCurBlockViewID(viewID) } -// SetLeaderIndex set the leader index. -func (consensus *Consensus) SetLeaderIndex(f func(int) int) (current int) { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() - consensus.LeaderIndex = f(consensus.LeaderIndex) - return consensus.LeaderIndex -} - -func (consensus *Consensus) GetLeaderIndex() int { - consensus.pubKeyLock.Lock() - defer consensus.pubKeyLock.Unlock() - return consensus.LeaderIndex -} // SetCurBlockViewID set the current view ID func (consensus *Consensus) setCurBlockViewID(viewID uint64) { consensus.current.SetCurBlockViewID(viewID) @@ -524,7 +511,6 @@ func (consensus *Consensus) SetViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) } - // SetViewChangingID set the current view change ID func (consensus *Consensus) setViewChangingID(viewID uint64) { consensus.current.SetViewChangingID(viewID) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 3ef0ea0655..23c51ba248 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -414,18 +414,18 @@ func (consensus *Consensus) tick() { func (consensus *Consensus) BlockChannel(newBlock *types.Block) { //consensus.ReshardingNextLeader(newBlock) - consensus.getLogger().Info(). - Uint64("MsgBlockNum", newBlock.NumberU64()). - Msg("[ConsensusMainLoop] Received Proposed New Block!") + consensus.getLogger().Info(). + Uint64("MsgBlockNum", newBlock.NumberU64()). + Msg("[ConsensusMainLoop] Received Proposed New Block!") if newBlock.NumberU64() < consensus.BlockNum() { consensus.getLogger().Warn().Uint64("newBlockNum", newBlock.NumberU64()). Msg("[ConsensusMainLoop] received old block, abort") return - } - // Sleep to wait for the full block time - consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") - time.AfterFunc(time.Until(consensus.NextBlockDue), func() { + } + // Sleep to wait for the full block time + consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") + time.AfterFunc(time.Until(consensus.NextBlockDue), func() { consensus.StartFinalityCount() consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -750,12 +750,6 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg } else { epoch = blk.Epoch() } - var epoch *big.Int - if blk.IsLastBlockInEpoch() { - epoch = new(big.Int).Add(blk.Epoch(), common.Big1) - } else { - epoch = blk.Epoch() - } if consensus.Blockchain.Config().IsLeaderRotation(epoch) { consensus.rotateLeader(epoch) } diff --git a/consensus/view_change.go b/consensus/view_change.go index 06640c123d..3ab3d2c9da 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -327,8 +327,8 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri consensus.current.SetMode(Normal) consensus.consensusTimeout[timeoutViewChange].Stop() - consensus.SetViewIDs(viewID) - consensus.ResetViewChangeState() + consensus.setViewIDs(viewID) + consensus.resetViewChangeState() consensus.consensusTimeout[timeoutConsensus].Start() consensus.getLogger().Info(). @@ -340,7 +340,6 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri if reset { consensus.resetState() } - fmt.Println("[startNewView]", newLeaderPriKey.Pub.Bytes.Hex()) consensus.setLeaderPubKey(newLeaderPriKey.Pub) return nil @@ -532,7 +531,7 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // newView message verified success, override my state consensus.setViewIDs(recvMsg.ViewID) consensus.LeaderPubKey = senderKey - consensus.ResetViewChangeState() + consensus.resetViewChangeState() consensus.msgSender.StopRetry(msg_pb.MessageType_VIEWCHANGE) @@ -553,6 +552,13 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { // ResetViewChangeState resets the view change structure func (consensus *Consensus) ResetViewChangeState() { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + consensus.resetViewChangeState() +} + +// ResetViewChangeState resets the view change structure +func (consensus *Consensus) resetViewChangeState() { consensus.getLogger().Info(). Str("Phase", consensus.phase.String()). Msg("[ResetViewChangeState] Resetting view change state") From 74c2e9bd7862d0b7ad6771833f26705c769c7e8d Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Fri, 27 Jan 2023 01:52:17 -0300 Subject: [PATCH 361/420] Rebased on dev. --- consensus/consensus_v2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 23c51ba248..44c3ebbd21 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -750,7 +750,7 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg } else { epoch = blk.Epoch() } - if consensus.Blockchain.Config().IsLeaderRotation(epoch) { + if consensus.Blockchain().Config().IsLeaderRotation(epoch) { consensus.rotateLeader(epoch) } From d64bfa784b18ec81aae81818571a1149434b3447 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Fri, 27 Jan 2023 19:14:09 -0300 Subject: [PATCH 362/420] Fix usage of private methods. --- consensus/checks.go | 3 ++- consensus/consensus_service.go | 13 +++++++++++-- consensus/consensus_v2.go | 5 +++-- consensus/validator.go | 2 +- consensus/view_change.go | 12 ++++++------ core/blockchain_impl.go | 15 +++++++++++++-- 6 files changed, 36 insertions(+), 14 deletions(-) diff --git a/consensus/checks.go b/consensus/checks.go index ff4b53927e..28da66ad7b 100644 --- a/consensus/checks.go +++ b/consensus/checks.go @@ -56,7 +56,8 @@ func (consensus *Consensus) senderKeySanityChecks(msg *msg_pb.Message, senderKey } func (consensus *Consensus) isRightBlockNumAndViewID(recvMsg *FBFTMessage) bool { - if recvMsg.ViewID != consensus.GetCurBlockViewID() || recvMsg.BlockNum != consensus.BlockNum() { + blockNum := consensus.getBlockNum() + if recvMsg.ViewID != consensus.getCurBlockViewID() || recvMsg.BlockNum != blockNum { consensus.getLogger().Debug(). Uint64("blockNum", blockNum). Str("recvMsg", recvMsg.String()). diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index a92a71f7e4..c43c7fe127 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -486,6 +486,8 @@ func (consensus *Consensus) isLeader() bool { // SetViewIDs set both current view ID and view changing ID to the height // of the blockchain. It is used during client startup to recover the state func (consensus *Consensus) SetViewIDs(height uint64) { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() consensus.setViewIDs(height) } @@ -625,11 +627,18 @@ func (consensus *Consensus) NumSignaturesIncludedInBlock(block *types.Block) uin return count } +// GetLogger returns logger for consensus contexts added. +func (consensus *Consensus) GetLogger() *zerolog.Logger { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() + return consensus.getLogger() +} + // getLogger returns logger for consensus contexts added func (consensus *Consensus) getLogger() *zerolog.Logger { logger := utils.Logger().With(). - Uint64("myBlock", consensus.BlockNum()). - Uint64("myViewID", consensus.GetCurBlockViewID()). + Uint64("myBlock", consensus.blockNum). + Uint64("myViewID", consensus.getCurBlockViewID()). Str("phase", consensus.phase.String()). Str("mode", consensus.current.Mode().String()). Logger() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 44c3ebbd21..1ab33d0bbf 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -46,6 +46,8 @@ const ( // IsViewChangingMode return true if curernt mode is viewchanging func (consensus *Consensus) IsViewChangingMode() bool { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() return consensus.isViewChangingMode() } @@ -413,8 +415,7 @@ func (consensus *Consensus) tick() { } func (consensus *Consensus) BlockChannel(newBlock *types.Block) { - //consensus.ReshardingNextLeader(newBlock) - consensus.getLogger().Info(). + consensus.GetLogger().Info(). Uint64("MsgBlockNum", newBlock.NumberU64()). Msg("[ConsensusMainLoop] Received Proposed New Block!") diff --git a/consensus/validator.go b/consensus/validator.go index 06aa9bc08c..f85cb8e3d8 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -277,7 +277,7 @@ func (consensus *Consensus) onPrepared(recvMsg *FBFTMessage) { if committedMsg != nil { consensus.onCommitted(committedMsg) } - if curBlockNum < consensus.BlockNum() { + if curBlockNum < consensus.getBlockNum() { consensus.getLogger().Info().Msg("[OnPrepared] Successfully caught up with committed message") break } diff --git a/consensus/view_change.go b/consensus/view_change.go index 3ab3d2c9da..13acb86ae2 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -253,7 +253,7 @@ func (consensus *Consensus) startViewChange() { consensus.getLogger().Warn(). Uint64("nextViewID", nextViewID). - Uint64("viewChangingID", consensus.GetViewChangingID()). + Uint64("viewChangingID", consensus.getViewChangingID()). Dur("timeoutDuration", duration). Str("NextLeader", consensus.LeaderPubKey.Bytes.Hex()). Msg("[startViewChange]") @@ -270,7 +270,7 @@ func (consensus *Consensus) startViewChange() { if err := consensus.vc.InitPayload( consensus.FBFTLog, nextViewID, - consensus.BlockNum(), + consensus.getBlockNum(), consensus.priKey, members); err != nil { consensus.getLogger().Error().Err(err).Msg("[startViewChange] Init Payload Error") @@ -284,7 +284,7 @@ func (consensus *Consensus) startViewChange() { } msgToSend := consensus.constructViewChangeMessage(&key) if err := consensus.msgSender.SendWithRetry( - consensus.BlockNum(), + consensus.getBlockNum(), msg_pb.MessageType_VIEWCHANGE, []nodeconfig.GroupID{ nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, @@ -310,7 +310,7 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri } if err := consensus.msgSender.SendWithRetry( - consensus.BlockNum(), + consensus.getBlockNum(), msg_pb.MessageType_NEWVIEW, []nodeconfig.GroupID{ nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, @@ -450,10 +450,10 @@ func (consensus *Consensus) onNewView(recvMsg *FBFTMessage) { Msg("[onNewView] Received NewView Message") // change view and leaderKey to keep in sync with network - if consensus.BlockNum() != recvMsg.BlockNum { + if consensus.getBlockNum() != recvMsg.BlockNum { consensus.getLogger().Warn(). Uint64("MsgBlockNum", recvMsg.BlockNum). - Uint64("myBlockNum", consensus.BlockNum()). + Uint64("myBlockNum", consensus.getBlockNum()). Msg("[onNewView] Invalid block number") return } diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 783a571e89..8b0683bd65 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -327,8 +327,19 @@ func VerifyBlockCrossLinks(blockchain BlockChain, block *types.Block) error { // ReadCrossLink beacon chain usage. cl, err := blockchain.ReadCrossLink(crossLink.ShardID(), crossLink.BlockNum()) if err == nil && cl != nil { - // Add slash for exist same blocknum but different crosslink - return errAlreadyExist + utils.Logger().Err(errAlreadyExist). + Uint64("beacon-block-number", block.NumberU64()). + Interface("remote", crossLink). + Interface("local", cl). + Msg("[CrossLinkVerification]") + // TODO Add slash for exist same blocknum but different crosslink + return errors.Wrapf( + errAlreadyExist, + "[CrossLinkVerification] shard: %d block: %d on beacon block %d", + crossLink.ShardID(), + crossLink.BlockNum(), + block.NumberU64(), + ) } if err := VerifyCrossLink(blockchain, crossLink); err != nil { return errors.Wrapf(err, "cannot VerifyBlockCrossLinks") From 18cbe84ef6d5d37edcecae7cedd99c2697da1c30 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sat, 28 Jan 2023 00:21:05 -0300 Subject: [PATCH 363/420] Fix usage of private methods. --- consensus/consensus_service.go | 12 ++++++++++++ consensus/consensus_v2.go | 16 ++++++++-------- consensus/view_change.go | 12 ++++++------ 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index c43c7fe127..1ce41392cc 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -210,6 +210,13 @@ func (consensus *Consensus) SetIsBackup(isBackup bool) { // Mode returns the mode of consensus func (consensus *Consensus) Mode() Mode { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() + return consensus.mode() +} + +// mode returns the mode of consensus +func (consensus *Consensus) mode() Mode { return consensus.current.Mode() } @@ -254,6 +261,11 @@ func (consensus *Consensus) SetBlockNum(blockNum uint64) { atomic.StoreUint64(&consensus.blockNum, blockNum) } +// SetBlockNum sets the blockNum in consensus object, called at node bootstrap +func (consensus *Consensus) setBlockNum(blockNum uint64) { + atomic.StoreUint64(&consensus.blockNum, blockNum) +} + // ReadSignatureBitmapPayload read the payload for signature and bitmap; offset is the beginning position of reading func (consensus *Consensus) ReadSignatureBitmapPayload(recvPayload []byte, offset int) (*bls_core.Sign, *bls_cosi.Mask, error) { consensus.mutex.RLock() diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 1ab33d0bbf..4244cb3732 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -367,7 +367,7 @@ func (consensus *Consensus) syncReadyChan() { func (consensus *Consensus) syncNotReadyChan() { consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncNotReadyChan") - consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) + consensus.setBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) consensus.current.SetMode(Syncing) consensus.getLogger().Info().Msg("[ConsensusMainLoop] Node is OUT OF SYNC") consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() @@ -452,14 +452,14 @@ func (consensus *Consensus) BlockChannel(newBlock *types.Block) { // waitForCommit wait extra 2 seconds for commit phase to finish func (consensus *Consensus) waitForCommit() { - if consensus.Mode() != Normal || consensus.phase.Get() != FBFTCommit { + if consensus.mode() != Normal || consensus.phase.Get() != FBFTCommit { return } // We only need to wait consensus is in normal commit phase utils.Logger().Warn().Str("phase", consensus.phase.String()).Msg("[shutdown] commit phase has to wait") maxWait := time.Now().Add(2 * consensus.BlockPeriod) - for time.Now().Before(maxWait) && consensus.GetConsensusPhase() == "Commit" { + for time.Now().Before(maxWait) && consensus.getConsensusPhase() == "Commit" { utils.Logger().Warn().Msg("[shutdown] wait for consensus finished") time.Sleep(time.Millisecond * 100) } @@ -635,7 +635,7 @@ func (consensus *Consensus) tryCatchup() error { if consensus.BlockVerifier == nil { return errors.New("consensus haven't finished initialization") } - initBN := consensus.BlockNum() + initBN := consensus.getBlockNum() defer consensus.postCatchup(initBN) blks, msgs, err := consensus.getLastMileBlocksAndMsg(initBN) @@ -764,15 +764,15 @@ func (consensus *Consensus) setupForNewConsensus(blk *types.Block, committedMsg } func (consensus *Consensus) postCatchup(initBN uint64) { - if initBN < consensus.BlockNum() { + if initBN < consensus.getBlockNum() { consensus.getLogger().Info(). Uint64("From", initBN). - Uint64("To", consensus.BlockNum()). + Uint64("To", consensus.getBlockNum()). Msg("[TryCatchup] Caught up!") consensus.switchPhase("TryCatchup", FBFTAnnounce) } // catch up and skip from view change trap - if initBN < consensus.BlockNum() && consensus.isViewChangingMode() { + if initBN < consensus.getBlockNum() && consensus.isViewChangingMode() { consensus.current.SetMode(Normal) consensus.consensusTimeout[timeoutViewChange].Stop() } @@ -833,7 +833,7 @@ func (consensus *Consensus) GenerateVdfAndProof(newBlock *types.Block, vrfBlockN start := time.Now() vdf.Execute() duration := time.Since(start) - consensus.getLogger().Info(). + consensus.GetLogger().Info(). Dur("duration", duration). Msg("[ConsensusMainLoop] VDF computation finished") output := <-outputChannel diff --git a/consensus/view_change.go b/consensus/view_change.go index 13acb86ae2..d8b1058a04 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -88,14 +88,14 @@ func (pm *State) SetIsBackup(isBackup bool) { // fallbackNextViewID return the next view ID and duration when there is an exception // to calculate the time-based viewId func (consensus *Consensus) fallbackNextViewID() (uint64, time.Duration) { - diff := int64(consensus.GetViewChangingID() + 1 - consensus.GetCurBlockViewID()) + diff := int64(consensus.getViewChangingID() + 1 - consensus.getCurBlockViewID()) if diff <= 0 { diff = int64(1) } consensus.getLogger().Error(). Int64("diff", diff). Msg("[fallbackNextViewID] use legacy viewID algorithm") - return consensus.GetViewChangingID() + 1, time.Duration(diff * diff * int64(viewChangeDuration)) + return consensus.getViewChangingID() + 1, time.Duration(diff * diff * int64(viewChangeDuration)) } // getNextViewID return the next view ID based on the timestamp @@ -152,7 +152,7 @@ func (consensus *Consensus) getNextViewID() (uint64, time.Duration) { func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrapper { gap := 1 - cur := consensus.GetCurBlockViewID() + cur := consensus.getCurBlockViewID() if viewID > cur { gap = int(viewID - cur) } @@ -196,7 +196,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe Str("leaderPubKey", consensus.LeaderPubKey.Bytes.Hex()). Int("gap", gap). Uint64("newViewID", viewID). - Uint64("myCurBlockViewID", consensus.GetCurBlockViewID()). + Uint64("myCurBlockViewID", consensus.getCurBlockViewID()). Msg("[getNextLeaderKey] got leaderPubKey from coinbase") // wasFound, next := consensus.Decider.NthNext(lastLeaderPubKey, gap) // FIXME: rotate leader on harmony nodes only before fully externalization @@ -234,7 +234,7 @@ func createTimeout() map[TimeoutType]*utils.Timeout { // startViewChange start the view change process func (consensus *Consensus) startViewChange() { - if consensus.disableViewChange || consensus.IsBackup() { + if consensus.disableViewChange || consensus.isBackup { return } @@ -242,7 +242,7 @@ func (consensus *Consensus) startViewChange() { consensus.consensusTimeout[timeoutBootstrap].Stop() consensus.current.SetMode(ViewChanging) nextViewID, duration := consensus.getNextViewID() - consensus.SetViewChangingID(nextViewID) + consensus.setViewChangingID(nextViewID) // TODO: set the Leader PubKey to the next leader for view change // this is dangerous as the leader change is not succeeded yet // we use it this way as in many code we validate the messages From eb6c946fe16139a8b7b3d22e703ea461e6dff5da Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Sat, 28 Jan 2023 01:11:28 -0300 Subject: [PATCH 364/420] Fix usage of private methods. --- consensus/consensus_service.go | 4 ++-- consensus/consensus_v2.go | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 1ce41392cc..71582df33a 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -248,9 +248,9 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { Str("leaderKey", consensus.LeaderPubKey.Bytes.Hex()). Msg("[checkViewID] Start consensus timer") return nil - } else if msg.ViewID > consensus.GetCurBlockViewID() { + } else if msg.ViewID > consensus.getCurBlockViewID() { return consensus_engine.ErrViewIDNotMatch - } else if msg.ViewID < consensus.GetCurBlockViewID() { + } else if msg.ViewID < consensus.getCurBlockViewID() { return errors.New("view ID belongs to the past") } return nil diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 4244cb3732..fe6b802b33 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -144,7 +144,7 @@ func (consensus *Consensus) finalCommit() { consensus.getLogger().Info(). Int64("NumCommits", numCommits). Msg("[finalCommit] Finalizing Consensus") - beforeCatchupNum := consensus.BlockNum() + beforeCatchupNum := consensus.getBlockNum() leaderPriKey, err := consensus.getConsensusLeaderPrivateKey() if err != nil { @@ -345,16 +345,16 @@ func (consensus *Consensus) StartChannel() { func (consensus *Consensus) syncReadyChan() { consensus.getLogger().Info().Msg("[ConsensusMainLoop] syncReadyChan") - if consensus.BlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { - consensus.SetBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) - consensus.SetViewIDs(consensus.Blockchain().CurrentHeader().ViewID().Uint64() + 1) + if consensus.getBlockNum() < consensus.Blockchain().CurrentHeader().Number().Uint64()+1 { + consensus.setBlockNum(consensus.Blockchain().CurrentHeader().Number().Uint64() + 1) + consensus.setViewIDs(consensus.Blockchain().CurrentHeader().ViewID().Uint64() + 1) mode := consensus.updateConsensusInformation() consensus.current.SetMode(mode) consensus.getLogger().Info().Msg("[syncReadyChan] Start consensus timer") consensus.consensusTimeout[timeoutConsensus].Start() consensus.getLogger().Info().Str("Mode", mode.String()).Msg("Node is IN SYNC") consensusSyncCounterVec.With(prometheus.Labels{"consensus": "in_sync"}).Inc() - } else if consensus.Mode() == Syncing { + } else if consensus.mode() == Syncing { // Corner case where sync is triggered before `onCommitted` and there is a race // for block insertion between consensus and downloader. mode := consensus.updateConsensusInformation() From 3af8a96cb3f92b301a7331f80967e1661702d9c5 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 31 Jan 2023 12:04:33 -0300 Subject: [PATCH 365/420] Removed deadcode, LockedFBFTPhase. --- consensus/consensus_service.go | 2 +- consensus/consensus_v2.go | 15 --------------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 71582df33a..0e4bb68141 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -554,7 +554,7 @@ func (consensus *Consensus) switchPhase(subject string, desired FBFTPhase) { Str("to:", desired.String()). Str("switchPhase:", subject) - consensus.phase.Set(desired) + consensus.phase = desired } var ( diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index fe6b802b33..3f9a2cfa2d 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -450,21 +450,6 @@ func (consensus *Consensus) BlockChannel(newBlock *types.Block) { } } -// waitForCommit wait extra 2 seconds for commit phase to finish -func (consensus *Consensus) waitForCommit() { - if consensus.mode() != Normal || consensus.phase.Get() != FBFTCommit { - return - } - // We only need to wait consensus is in normal commit phase - utils.Logger().Warn().Str("phase", consensus.phase.String()).Msg("[shutdown] commit phase has to wait") - - maxWait := time.Now().Add(2 * consensus.BlockPeriod) - for time.Now().Before(maxWait) && consensus.getConsensusPhase() == "Commit" { - utils.Logger().Warn().Msg("[shutdown] wait for consensus finished") - time.Sleep(time.Millisecond * 100) - } -} - // LastMileBlockIter is the iterator to iterate over the last mile blocks in consensus cache. // All blocks returned are guaranteed to pass the verification. type LastMileBlockIter struct { From a1cda807a42699de5c1b9aac7a744c2c9e20b03a Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 31 Jan 2023 12:12:43 -0300 Subject: [PATCH 366/420] Fix review comment. --- consensus/consensus_v2.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 3f9a2cfa2d..6b0d54648d 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -310,9 +310,7 @@ func (consensus *Consensus) Start( case <-stopChan: return case <-ticker.C: - consensus.mutex.Lock() - consensus.tick() - consensus.mutex.Unlock() + consensus.Tick() } } }() @@ -373,6 +371,12 @@ func (consensus *Consensus) syncNotReadyChan() { consensusSyncCounterVec.With(prometheus.Labels{"consensus": "out_of_sync"}).Inc() } +func (consensus *Consensus) Tick() { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + consensus.tick() +} + func (consensus *Consensus) tick() { if !consensus.start && consensus.isInitialLeader { return From e87bd58b229d22a55c2e4f7e665aebad810d0101 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 31 Jan 2023 14:29:17 -0300 Subject: [PATCH 367/420] Go mod tidy. --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index 6bbfc03980..18e16a0c05 100644 --- a/go.mod +++ b/go.mod @@ -166,7 +166,6 @@ require ( github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect - github.com/libp2p/go-libp2p-core v0.20.1 // indirect github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect github.com/libp2p/go-msgio v0.2.0 // indirect From adab3d6d290ca31bc6713dffe9c08bb5952e7239 Mon Sep 17 00:00:00 2001 From: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Date: Thu, 23 Feb 2023 09:24:23 +0000 Subject: [PATCH 368/420] remove default timeouts --- rosetta/infra/harmony-mainnet.conf | 2 -- rosetta/infra/harmony-pstn.conf | 2 -- 2 files changed, 4 deletions(-) diff --git a/rosetta/infra/harmony-mainnet.conf b/rosetta/infra/harmony-mainnet.conf index 6460075607..8d51609cb1 100644 --- a/rosetta/infra/harmony-mainnet.conf +++ b/rosetta/infra/harmony-mainnet.conf @@ -74,8 +74,6 @@ Version = "2.5.13" MaxConnsPerIP = 10 MaxPeers = 0 Port = 9000 - ConnManagerLowWatermark = 160 - ConnManagerHighWatermark = 192 WaitForEachPeerToConnect = false [Pprof] diff --git a/rosetta/infra/harmony-pstn.conf b/rosetta/infra/harmony-pstn.conf index da2ab27847..1bb865c1a5 100644 --- a/rosetta/infra/harmony-pstn.conf +++ b/rosetta/infra/harmony-pstn.conf @@ -74,8 +74,6 @@ Version = "2.5.13" MaxConnsPerIP = 10 MaxPeers = 0 Port = 9000 - ConnManagerLowWatermark = 160 - ConnManagerHighWatermark = 192 WaitForEachPeerToConnect = false [Pprof] From f6e5b84232f07a3f0787a15067e2a6fdccc721b1 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 21 Feb 2023 18:45:17 -0300 Subject: [PATCH 369/420] Rotate external leaders on non-beacon chains. --- consensus/consensus_v2.go | 10 +++++++++- consensus/quorum/quorum.go | 1 + consensus/view_change.go | 16 +++++++++++----- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 6b0d54648d..99bccf7554 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -714,7 +714,15 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { } } // Passed all checks, we can change leader. - wasFound, next := consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) + var ( + wasFound bool + next *bls.PublicKeyWrapper + ) + if consensus.ShardID == shard.BeaconChainShardID { + wasFound, next = consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) + } else { + wasFound, next = consensus.Decider.NthNext(leader, 1) + } if !wasFound { utils.Logger().Error().Msg("Failed to get next leader") return diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index da1551cdf9..aba62fa53f 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -75,6 +75,7 @@ type ParticipantTracker interface { Participants() multibls.PublicKeys IndexOf(bls.SerializedPublicKey) int ParticipantsCount() int64 + NthNext(*bls.PublicKeyWrapper, int) (bool, *bls.PublicKeyWrapper) NthNextHmy(shardingconfig.Instance, *bls.PublicKeyWrapper, int) (bool, *bls.PublicKeyWrapper) NthNextHmyExt(shardingconfig.Instance, *bls.PublicKeyWrapper, int) (bool, *bls.PublicKeyWrapper) FirstParticipant(shardingconfig.Instance) *bls.PublicKeyWrapper diff --git a/consensus/view_change.go b/consensus/view_change.go index d8b1058a04..899a50121d 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -202,11 +202,17 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe // FIXME: rotate leader on harmony nodes only before fully externalization var wasFound bool var next *bls.PublicKeyWrapper - if blockchain != nil && blockchain.Config().IsAllowlistEpoch(epoch) { - wasFound, next = consensus.Decider.NthNextHmyExt( - shard.Schedule.InstanceForEpoch(epoch), - lastLeaderPubKey, - gap) + if consensus.Blockchain().Config().IsLeaderRotation(epoch) { + if consensus.ShardID == shard.BeaconChainShardID { + wasFound, next = consensus.Decider.NthNextHmy( + shard.Schedule.InstanceForEpoch(epoch), + lastLeaderPubKey, + gap) + } else { + wasFound, next = consensus.Decider.NthNext( + lastLeaderPubKey, + gap) + } } else { wasFound, next = consensus.Decider.NthNextHmy( shard.Schedule.InstanceForEpoch(epoch), From e6287c16350890549b61a1982f72fa424eb5f904 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 21 Feb 2023 22:33:57 -0300 Subject: [PATCH 370/420] Fix nil panic. --- consensus/view_change.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/view_change.go b/consensus/view_change.go index 899a50121d..aafdfd1210 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -202,7 +202,7 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe // FIXME: rotate leader on harmony nodes only before fully externalization var wasFound bool var next *bls.PublicKeyWrapper - if consensus.Blockchain().Config().IsLeaderRotation(epoch) { + if blockchain != nil && blockchain.Config().IsLeaderRotation(epoch) { if consensus.ShardID == shard.BeaconChainShardID { wasFound, next = consensus.Decider.NthNextHmy( shard.Schedule.InstanceForEpoch(epoch), From 75df86692ade4c513656fe36ac6e091cc994ad39 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 14 Mar 2023 21:13:31 -0300 Subject: [PATCH 371/420] Fixes. --- consensus/consensus.go | 14 +++----------- internal/params/config.go | 1 - 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 6d396c100c..e0cc591dfe 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -187,18 +187,10 @@ func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { return consensus.LeaderPubKey } -func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { - return consensus.LeaderPubKey -} - -func (consensus *Consensus) getLeaderPubKey() *bls_cosi.PublicKeyWrapper { - return consensus.LeaderPubKey -} - func (consensus *Consensus) SetLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { - consensus.pubKeyLock.Lock() - consensus.LeaderPubKey = pub - consensus.pubKeyLock.Unlock() + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + consensus.setLeaderPubKey(pub) } func (consensus *Consensus) setLeaderPubKey(pub *bls_cosi.PublicKeyWrapper) { diff --git a/internal/params/config.go b/internal/params/config.go index fceca81627..b50aa35532 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -111,7 +111,6 @@ var ( AllowlistEpoch: big.NewInt(2), LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 64, - TesnetNinetyPercentEpoch: big.NewInt(399), FeeCollectEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. From 3f0a11638b1209bb52267f226cf1179d1ad4224f Mon Sep 17 00:00:00 2001 From: Casey Gardiner <117784577+ONECasey@users.noreply.github.com> Date: Wed, 15 Mar 2023 11:03:39 -0700 Subject: [PATCH 372/420] Update singleton.go --- internal/utils/singleton.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/utils/singleton.go b/internal/utils/singleton.go index 1b71e981b5..10101d7673 100644 --- a/internal/utils/singleton.go +++ b/internal/utils/singleton.go @@ -197,7 +197,6 @@ func updateZeroLogLevel(level int) { zeroLogger = &childLogger } - // GetPort is useful for debugging, returns `--port` flag provided to executable. func GetPort() int { ok := false From ab4a10547accb95015456e5808c36fabe9c103eb Mon Sep 17 00:00:00 2001 From: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Date: Thu, 16 Mar 2023 21:12:35 +0000 Subject: [PATCH 373/420] evm: don't return extcode for validators Due to technical debt, validator information is stored in the code field of the address. The code field can be accessed in Solidity for an arbitrary address using `extcodesize`, `extcodehash`, and `extcodecopy` or helper commands (such as `address.code.Length`). The presence of this field is used by contract developers to (erroneously) deny smart contract access to other smart contracts (and therefore, validators). This PR fixes that oversight by returning the same values as other EOAs for known validator addresses. Obviously, it needs a hard fork that will be scheduled separately. --- core/vm/instructions.go | 32 ++++++++++++++++++++++++++++++-- internal/params/config.go | 23 +++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 2c1c3154ac..091ba28ff6 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common/math" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/params" + "github.com/harmony-one/harmony/shard" "golang.org/x/crypto/sha3" ) @@ -477,6 +478,16 @@ func opReturnDataCopy(pc *uint64, interpreter *EVMInterpreter, contract *Contrac func opExtCodeSize(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { slot := stack.peek() + address := common.BigToAddress(slot) + fixValidatorCode := interpreter.evm.chainRules.IsValidatorCodeFix && + interpreter.evm.ShardID == shard.BeaconChainShardID && + interpreter.evm.StateDB.IsValidator(address) + if fixValidatorCode { + // https://github.com/ethereum/solidity/blob/develop/Changelog.md#081-2021-01-27 + // per this link,
.code.length calls extcodesize on the address so this fix will work + slot.SetUint64(0) + return nil, nil + } slot.SetUint64(uint64(interpreter.evm.StateDB.GetCodeSize(common.BigToAddress(slot)))) return nil, nil @@ -509,7 +520,17 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, contract *Contract, codeOffset = stack.pop() length = stack.pop() ) - codeCopy := getDataBig(interpreter.evm.StateDB.GetCode(addr), codeOffset, length) + var code []byte + fixValidatorCode := interpreter.evm.chainRules.IsValidatorCodeFix && + interpreter.evm.ShardID == shard.BeaconChainShardID && + interpreter.evm.StateDB.IsValidator(addr) + if fixValidatorCode { + // for EOAs that are not validators, statedb returns nil + code = nil + } else { + code = interpreter.evm.StateDB.GetCode(addr) + } + codeCopy := getDataBig(code, codeOffset, length) memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy) interpreter.intPool.put(memOffset, codeOffset, length) @@ -555,7 +576,14 @@ func opExtCodeHash(pc *uint64, interpreter *EVMInterpreter, contract *Contract, if interpreter.evm.StateDB.Empty(address) { slot.SetUint64(0) } else { - slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(address).Bytes()) + fixValidatorCode := interpreter.evm.chainRules.IsValidatorCodeFix && + interpreter.evm.ShardID == shard.BeaconChainShardID && + interpreter.evm.StateDB.IsValidator(address) + if fixValidatorCode { + slot.SetBytes(emptyCodeHash.Bytes()) + } else { + slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(address).Bytes()) + } } return nil, nil } diff --git a/internal/params/config.go b/internal/params/config.go index b50aa35532..d900386303 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -72,6 +72,7 @@ var ( FeeCollectEpoch: EpochTBD, LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 64, + ValidatorCodeFixEpoch: EpochTBD, } // TestnetChainConfig contains the chain parameters to run a node on the harmony test network. @@ -112,6 +113,7 @@ var ( LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 64, FeeCollectEpoch: EpochTBD, + ValidatorCodeFixEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. // All features except for CrossLink are enabled at launch. @@ -152,6 +154,7 @@ var ( LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 64, FeeCollectEpoch: EpochTBD, + ValidatorCodeFixEpoch: EpochTBD, } // PartnerChainConfig contains the chain parameters for the Partner network. @@ -193,6 +196,7 @@ var ( FeeCollectEpoch: big.NewInt(574), LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 64, + ValidatorCodeFixEpoch: EpochTBD, } // StressnetChainConfig contains the chain parameters for the Stress test network. @@ -234,6 +238,7 @@ var ( FeeCollectEpoch: EpochTBD, LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 64, + ValidatorCodeFixEpoch: EpochTBD, } // LocalnetChainConfig contains the chain parameters to run for local development. @@ -274,6 +279,7 @@ var ( LeaderRotationEpoch: EpochTBD, LeaderRotationBlocksCount: 5, FeeCollectEpoch: big.NewInt(5), + ValidatorCodeFixEpoch: EpochTBD, } // AllProtocolChanges ... @@ -316,6 +322,7 @@ var ( big.NewInt(1), // LeaderRotationEpoch 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch + big.NewInt(0), // ValidatorCodeFixEpoch } // TestChainConfig ... @@ -358,6 +365,7 @@ var ( big.NewInt(1), // LeaderRotationEpoch 64, // LeaderRotationBlocksCount big.NewInt(0), // FeeCollectEpoch + big.NewInt(0), // ValidatorCodeFixEpoch } // TestRules ... @@ -507,6 +515,13 @@ type ChainConfig struct { // Then before FeeCollectEpoch, txn fees are burned. // After FeeCollectEpoch, txn fees paid to FeeCollector account. FeeCollectEpoch *big.Int + + // ValidatorCodeFixEpoch is the first epoch that fixes the issue of validator code + // being available in Solidity. This is a temporary fix until we have a better + // solution. + // Contracts can check the (presence of) validator code by calling the following: + // extcodesize, extcodecopy and extcodehash. + ValidatorCodeFixEpoch *big.Int `json:"validator-code-fix-epoch,omitempty"` } // String implements the fmt.Stringer interface. @@ -541,6 +556,8 @@ func (c *ChainConfig) mustValid() { "must satisfy: StakingPrecompileEpoch >= PreStakingEpoch") require(c.CrossShardXferPrecompileEpoch.Cmp(c.CrossTxEpoch) > 0, "must satisfy: CrossShardXferPrecompileEpoch > CrossTxEpoch") + require(c.ValidatorCodeFixEpoch.Cmp(c.EthCompatibleEpoch) >= 0, + "must satisfy: ValidatorCodeFixEpoch >= EthCompatibleEpoch") } // IsEIP155 returns whether epoch is either equal to the EIP155 fork epoch or greater. @@ -716,6 +733,10 @@ func (c *ChainConfig) IsFeeCollectEpoch(epoch *big.Int) bool { return isForked(c.FeeCollectEpoch, epoch) } +func (c *ChainConfig) IsValidatorCodeFix(epoch *big.Int) bool { + return isForked(c.ValidatorCodeFixEpoch, epoch) +} + // UpdateEthChainIDByShard update the ethChainID based on shard ID. func UpdateEthChainIDByShard(shardID uint32) { once.Do(func() { @@ -773,6 +794,7 @@ type Rules struct { IsStakingPrecompile, IsCrossShardXferPrecompile, // eip-155 chain id fix IsChainIdFix bool + IsValidatorCodeFix bool } // Rules ensures c's ChainID is not nil. @@ -797,5 +819,6 @@ func (c *ChainConfig) Rules(epoch *big.Int) Rules { IsStakingPrecompile: c.IsStakingPrecompile(epoch), IsCrossShardXferPrecompile: c.IsCrossShardXferPrecompile(epoch), IsChainIdFix: c.IsChainIdFix(epoch), + IsValidatorCodeFix: c.IsValidatorCodeFix(epoch), } } From d13f4d033db836f0d0e7dc64333a8faefed7482e Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 21 Mar 2023 23:51:52 -0300 Subject: [PATCH 374/420] Fix context passing. --- hmy/downloader/longrange.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/hmy/downloader/longrange.go b/hmy/downloader/longrange.go index b4efbf1926..4d4935b8f2 100644 --- a/hmy/downloader/longrange.go +++ b/hmy/downloader/longrange.go @@ -149,9 +149,8 @@ func (lsi *lrSyncIter) fetchAndInsertBlocks(targetBN uint64) error { worker := &getBlocksWorker{ gbm: gbm, protocol: lsi.p, - ctx: lsi.ctx, } - go worker.workLoop() + go worker.workLoop(lsi.ctx) } // insert the blocks to chain. Return when the target block number is reached. @@ -243,28 +242,26 @@ func (lsi *lrSyncIter) checkHaveEnoughStreams() error { type getBlocksWorker struct { gbm *getBlocksManager protocol syncProtocol - - ctx context.Context } -func (w *getBlocksWorker) workLoop() { +func (w *getBlocksWorker) workLoop(ctx context.Context) { for { select { - case <-w.ctx.Done(): + case <-ctx.Done(): return default: } batch := w.gbm.GetNextBatch() if len(batch) == 0 { select { - case <-w.ctx.Done(): + case <-ctx.Done(): return case <-time.After(100 * time.Millisecond): continue } } - blocks, stid, err := w.doBatch(batch) + blocks, stid, err := w.doBatch(ctx, batch) if err != nil { if !errors.Is(err, context.Canceled) { w.protocol.RemoveStream(stid) @@ -277,8 +274,8 @@ func (w *getBlocksWorker) workLoop() { } } -func (w *getBlocksWorker) doBatch(bns []uint64) ([]*types.Block, sttypes.StreamID, error) { - ctx, cancel := context.WithTimeout(w.ctx, 10*time.Second) +func (w *getBlocksWorker) doBatch(ctx context.Context, bns []uint64) ([]*types.Block, sttypes.StreamID, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() blocks, stid, err := w.protocol.GetBlocksByNumber(ctx, bns) From 47386c06a5df43a17ab23f9224637a4f53dd5c0e Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 22 Mar 2023 00:06:39 -0300 Subject: [PATCH 375/420] Clean up code. --- node/node.go | 2 +- node/node_syncing.go | 28 +++++----------------------- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/node/node.go b/node/node.go index ba398d6289..13988e2ad3 100644 --- a/node/node.go +++ b/node/node.go @@ -1029,6 +1029,7 @@ func New( unixTimeAtNodeStart: time.Now().Unix(), TransactionErrorSink: types.NewTransactionErrorSink(), crosslinks: crosslinks.New(), + syncID: GenerateSyncID(), } // Get the node config that's created in the harmony.go program. @@ -1039,7 +1040,6 @@ func New( } node.HarmonyConfig = harmonyconfig - copy(node.syncID[:], GenerateRandomString(SyncIDLength)) if host != nil { node.host = host node.SelfPeer = host.GetSelfPeer() diff --git a/node/node_syncing.go b/node/node_syncing.go index a0a8e6c84c..05ddd4b6b7 100644 --- a/node/node_syncing.go +++ b/node/node_syncing.go @@ -5,7 +5,6 @@ import ( "math/rand" "net" "strconv" - "sync" "time" "github.com/harmony-one/harmony/internal/tikv" @@ -65,28 +64,11 @@ func GenerateRandomString(n int) string { return string(b) } -// getNeighborPeers is a helper function to return list of peers -// based on different neightbor map -func getNeighborPeers(neighbor *sync.Map) []p2p.Peer { - tmp := []p2p.Peer{} - neighbor.Range(func(k, v interface{}) bool { - p := v.(p2p.Peer) - t := p.Port - p.Port = legacysync.GetSyncingPort(t) - tmp = append(tmp, p) - return true - }) - return tmp -} - -// DoSyncWithoutConsensus gets sync-ed to blockchain without joining consensus -func (node *Node) DoSyncWithoutConsensus() { - go node.DoSyncing(node.Blockchain(), node.Worker, false) //Don't join consensus -} - -// IsSameHeight tells whether node is at same bc height as a peer -func (node *Node) IsSameHeight() (uint64, bool) { - return node.SyncInstance().IsSameBlockchainHeight(node.Blockchain()) +// GenerateSyncID generates a random string with given length +func GenerateSyncID() [SyncIDLength]byte { + var syncID [SyncIDLength]byte + copy(syncID[:], GenerateRandomString(SyncIDLength)) + return syncID } func (node *Node) createStateSync(bc core.BlockChain) *legacysync.StateSync { From 8c2ff803f709f944cfc8b1278f35cf5b2cacf859 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 21 Mar 2023 00:49:36 -0300 Subject: [PATCH 376/420] Removed engine dependency. --- api/service/legacysync/syncing.go | 4 +-- api/service/stagedstreamsync/adapter.go | 1 - api/service/stagedstreamsync/sig_verify.go | 4 +-- api/service/stagedsync/stage_state.go | 2 +- api/service/stagedsync/stagedsync.go | 4 +-- block/header.go | 5 ++++ cmd/harmony/main.go | 7 ++--- consensus/engine/errors.go | 7 ----- consensus/validator.go | 2 +- core/block_validator.go | 16 ++++++------ core/blockchain.go | 2 -- core/blockchain_impl.go | 30 ++++++++++------------ core/epochchain.go | 10 +------- core/epochchain_test.go | 2 +- core/evm.go | 4 --- core/evm_test.go | 4 +-- core/headerchain.go | 10 ++------ core/state_processor.go | 18 ++++++------- core/tx_pool_test.go | 5 +--- hmy/downloader/adapter.go | 1 - hmy/downloader/downloader.go | 4 +-- hmy/tracer.go | 7 ++--- internal/chain/engine.go | 8 +++++- internal/shardchain/shardchains.go | 11 +++----- node/node.go | 4 +-- node/node_cross_link.go | 3 ++- node/node_handler_test.go | 17 +++++------- node/node_newblock_test.go | 6 ++--- node/node_test.go | 6 ++--- node/worker/worker.go | 8 +++--- node/worker/worker_test.go | 11 +++----- test/chain/chain/chain_makers.go | 13 +++++----- test/chain/main.go | 6 ++--- test/chain/reward/main.go | 4 +-- 34 files changed, 99 insertions(+), 147 deletions(-) diff --git a/api/service/legacysync/syncing.go b/api/service/legacysync/syncing.go index 3375ccdc50..07ef9604e3 100644 --- a/api/service/legacysync/syncing.go +++ b/api/service/legacysync/syncing.go @@ -880,12 +880,12 @@ func (ss *StateSync) UpdateBlockAndStatus(block *types.Block, bc core.BlockChain } startTime := time.Now() - if err := bc.Engine().VerifyHeaderSignature(bc, block.Header(), sig, bitmap); err != nil { + if err := chain.Engine().VerifyHeaderSignature(bc, block.Header(), sig, bitmap); err != nil { return errors.Wrapf(err, "verify header signature %v", block.Hash().String()) } utils.Logger().Debug().Int64("elapsed time", time.Now().Sub(startTime).Milliseconds()).Msg("[Sync] VerifyHeaderSignature") } - err := bc.Engine().VerifyHeader(bc, block.Header(), verifySeal) + err := chain.Engine().VerifyHeader(bc, block.Header(), verifySeal) if err == engine.ErrUnknownAncestor { return err } else if err != nil { diff --git a/api/service/stagedstreamsync/adapter.go b/api/service/stagedstreamsync/adapter.go index ae7632889c..bbd770c9b0 100644 --- a/api/service/stagedstreamsync/adapter.go +++ b/api/service/stagedstreamsync/adapter.go @@ -27,7 +27,6 @@ type syncProtocol interface { type blockChain interface { engine.ChainReader - Engine() engine.Engine InsertChain(chain types.Blocks, verifyHeaders bool) (int, error) WriteCommitSig(blockNum uint64, lastCommits []byte) error diff --git a/api/service/stagedstreamsync/sig_verify.go b/api/service/stagedstreamsync/sig_verify.go index 649c6eaec1..e50666bbde 100644 --- a/api/service/stagedstreamsync/sig_verify.go +++ b/api/service/stagedstreamsync/sig_verify.go @@ -47,10 +47,10 @@ func verifyAndInsertBlock(bc blockChain, block *types.Block, nextBlocks ...*type } } - if err := bc.Engine().VerifyHeaderSignature(bc, block.Header(), sigBytes, bitmap); err != nil { + if err := chain.Engine().VerifyHeaderSignature(bc, block.Header(), sigBytes, bitmap); err != nil { return &sigVerifyErr{err} } - if err := bc.Engine().VerifyHeader(bc, block.Header(), true); err != nil { + if err := chain.Engine().VerifyHeader(bc, block.Header(), true); err != nil { return errors.Wrap(err, "[VerifyHeader]") } if _, err := bc.InsertChain(types.Blocks{block}, false); err != nil { diff --git a/api/service/stagedsync/stage_state.go b/api/service/stagedsync/stage_state.go index 7086acec18..5492b890b5 100644 --- a/api/service/stagedsync/stage_state.go +++ b/api/service/stagedsync/stage_state.go @@ -250,7 +250,7 @@ func (stg *StageStates) verifyBlockSignatures(bc core.BlockChain, block *types.B } startTime := time.Now() - if err := bc.Engine().VerifyHeaderSignature(bc, block.Header(), sig, bitmap); err != nil { + if err := chain.Engine().VerifyHeaderSignature(bc, block.Header(), sig, bitmap); err != nil { return errors.Wrapf(err, "verify header signature %v", block.Hash().String()) } utils.Logger().Debug(). diff --git a/api/service/stagedsync/stagedsync.go b/api/service/stagedsync/stagedsync.go index 83af6abf9f..1080c97345 100644 --- a/api/service/stagedsync/stagedsync.go +++ b/api/service/stagedsync/stagedsync.go @@ -1052,14 +1052,14 @@ func (ss *StagedSync) UpdateBlockAndStatus(block *types.Block, bc core.BlockChai } startTime := time.Now() - if err := bc.Engine().VerifyHeaderSignature(bc, block.Header(), sig, bitmap); err != nil { + if err := chain.Engine().VerifyHeaderSignature(bc, block.Header(), sig, bitmap); err != nil { return errors.Wrapf(err, "verify header signature %v", block.Hash().String()) } utils.Logger().Debug(). Int64("elapsed time", time.Now().Sub(startTime).Milliseconds()). Msg("[STAGED_SYNC] VerifyHeaderSignature") } - err := bc.Engine().VerifyHeader(bc, block.Header(), verifySeal) + err := chain.Engine().VerifyHeader(bc, block.Header(), verifySeal) if err == engine.ErrUnknownAncestor { return err } else if err != nil { diff --git a/block/header.go b/block/header.go index 5672446cc7..6991825430 100644 --- a/block/header.go +++ b/block/header.go @@ -153,6 +153,11 @@ func (h *Header) IsLastBlockInEpoch() bool { return len(h.ShardState()) > 0 } +// NumberU64 returns the block number in uint64. +func (h Header) NumberU64() uint64 { + return h.Number().Uint64() +} + // HeaderRegistry is the taggedrlp type registry for versioned headers. var HeaderRegistry = taggedrlp.NewRegistry() diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index f01cb758ed..d13ba266e9 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -16,7 +16,6 @@ import ( "time" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/internal/registry" "github.com/harmony-one/harmony/internal/shardchain/tikv_manage" "github.com/harmony-one/harmony/internal/tikv/redis_helper" @@ -701,11 +700,9 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi chainDBFactory = &shardchain.LDBFactory{RootDir: nodeConfig.DBDir} } - engine := chain.NewEngine() - chainConfig := nodeConfig.GetNetworkType().ChainConfig() collection := shardchain.NewCollection( - &hc, chainDBFactory, &core.GenesisInitializer{NetworkType: nodeConfig.GetNetworkType()}, engine, &chainConfig, + &hc, chainDBFactory, &core.GenesisInitializer{NetworkType: nodeConfig.GetNetworkType()}, &chainConfig, ) for shardID, archival := range nodeConfig.ArchiveModes() { if archival { @@ -740,7 +737,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi os.Exit(1) } - currentNode := node.New(myHost, currentConsensus, engine, collection, blacklist, allowedTxs, localAccounts, nodeConfig.ArchiveModes(), &hc, registry) + currentNode := node.New(myHost, currentConsensus, collection, blacklist, allowedTxs, localAccounts, nodeConfig.ArchiveModes(), &hc, registry) if hc.Legacy != nil && hc.Legacy.TPBroadcastInvalidTxn != nil { currentNode.BroadcastInvalidTx = *hc.Legacy.TPBroadcastInvalidTxn diff --git a/consensus/engine/errors.go b/consensus/engine/errors.go index fe8e55f7bc..aab9614e9b 100644 --- a/consensus/engine/errors.go +++ b/consensus/engine/errors.go @@ -31,13 +31,6 @@ var ( // to the current node. ErrFutureBlock = errors.New("block in the future") - // ErrInvalidNumber is returned if a block's number doesn't equal it's parent's - // plus one. - ErrInvalidNumber = errors.New("invalid block number") - // ErrViewIDNotMatch is returned if the current viewID is not equal message's viewID ErrViewIDNotMatch = errors.New("viewID not match") - - // ErrInvalidConsensusMessage is returned is the consensus message received is invalid - ErrInvalidConsensusMessage = errors.New("invalid consensus message") ) diff --git a/consensus/validator.go b/consensus/validator.go index f85cb8e3d8..a270c665c6 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -331,7 +331,7 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { Msg("[OnCommitted] Failed to parse commit sigBytes and bitmap") return } - if err := consensus.Blockchain().Engine().VerifyHeaderSignature(consensus.Blockchain(), blockObj.Header(), + if err := chain.Engine().VerifyHeaderSignature(consensus.Blockchain(), blockObj.Header(), sigBytes, bitmap); err != nil { consensus.getLogger().Error(). Uint64("blockNum", recvMsg.BlockNum). diff --git a/core/block_validator.go b/core/block_validator.go index b1126f3972..4a590c81e6 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -26,6 +26,8 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/block" + "github.com/harmony-one/harmony/internal/chain" + chain2 "github.com/harmony-one/harmony/internal/chain" "github.com/pkg/errors" consensus_engine "github.com/harmony-one/harmony/consensus/engine" @@ -40,16 +42,14 @@ import ( // // BlockValidator implements validator. type BlockValidator struct { - config *params.ChainConfig // Chain configuration options - bc BlockChain // Canonical blockchain - engine consensus_engine.Engine // Consensus engine used for validating + config *params.ChainConfig // Chain configuration options + bc BlockChain // Canonical blockchain } // NewBlockValidator returns a new block validator which is safe for re-use -func NewBlockValidator(config *params.ChainConfig, blockchain BlockChain, engine consensus_engine.Engine) *BlockValidator { +func NewBlockValidator(config *params.ChainConfig, blockchain BlockChain) *BlockValidator { validator := &BlockValidator{ config: config, - engine: engine, bc: blockchain, } return validator @@ -131,7 +131,7 @@ func (v *BlockValidator) ValidateHeader(block *types.Block, seal bool) error { return errors.New("block is nil") } if h := block.Header(); h != nil { - return v.engine.VerifyHeader(v.bc, h, true) + return chain.Engine().VerifyHeader(v.bc, h, true) } return errors.New("header field was nil") } @@ -147,7 +147,7 @@ func (v *BlockValidator) ValidateHeaders(chain []*types.Block) (chan<- struct{}, headers[i] = block.Header() seals[i] = true } - return v.engine.VerifyHeaders(v.bc, headers, seals) + return chain2.Engine().VerifyHeaders(v.bc, headers, seals) } // CalcGasLimit computes the gas limit of the next block after parent. It aims @@ -249,5 +249,5 @@ func (v *BlockValidator) ValidateCXReceiptsProof(cxp *types.CXReceiptsProof) err // (4) verify blockHeader with seal var commitSig bls.SerializedSignature copy(commitSig[:], cxp.CommitSig) - return v.engine.VerifyHeaderSignature(v.bc, cxp.Header, commitSig, cxp.CommitBitmap) + return chain.Engine().VerifyHeaderSignature(v.bc, cxp.Header, commitSig, cxp.CommitBitmap) } diff --git a/core/blockchain.go b/core/blockchain.go index fda4831655..a28df6bab1 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -126,8 +126,6 @@ type BlockChain interface { GetHeaderByNumber(number uint64) *block.Header // Config retrieves the blockchain's chain configuration. Config() *params.ChainConfig - // Engine retrieves the blockchain's consensus engine. - Engine() engine.Engine // SubscribeRemovedLogsEvent registers a subscription of RemovedLogsEvent. SubscribeRemovedLogsEvent(ch chan<- RemovedLogsEvent) event.Subscription // SubscribeTraceEvent registers a subscription of ChainEvent. diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 8b0683bd65..5ed1224207 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -49,6 +49,8 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/crypto/bls" + "github.com/harmony-one/harmony/internal/chain" + chain2 "github.com/harmony-one/harmony/internal/chain" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/tikv" @@ -190,7 +192,6 @@ type BlockChainImpl struct { // procInterrupt must be atomically called procInterrupt int32 // interrupt signaler for block processing - engine consensus_engine.Engine processor Processor // block processor interface validator Validator // block and state validator interface vmConfig vm.Config @@ -204,9 +205,9 @@ type BlockChainImpl struct { // NewBlockChainWithOptions same as NewBlockChain but can accept additional behaviour options. func NewBlockChainWithOptions( db ethdb.Database, stateCache state.Database, beaconChain BlockChain, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, - engine consensus_engine.Engine, vmConfig vm.Config, options Options, + vmConfig vm.Config, options Options, ) (*BlockChainImpl, error) { - return newBlockChainWithOptions(db, stateCache, beaconChain, cacheConfig, chainConfig, engine, vmConfig, options) + return newBlockChainWithOptions(db, stateCache, beaconChain, cacheConfig, chainConfig, vmConfig, options) } // NewBlockChain returns a fully initialised block chain using information @@ -214,15 +215,15 @@ func NewBlockChainWithOptions( // Processor. func NewBlockChain( db ethdb.Database, stateCache state.Database, beaconChain BlockChain, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, - engine consensus_engine.Engine, vmConfig vm.Config, + vmConfig vm.Config, ) (*BlockChainImpl, error) { - return newBlockChainWithOptions(db, stateCache, beaconChain, cacheConfig, chainConfig, engine, vmConfig, Options{}) + return newBlockChainWithOptions(db, stateCache, beaconChain, cacheConfig, chainConfig, vmConfig, Options{}) } func newBlockChainWithOptions( db ethdb.Database, stateCache state.Database, beaconChain BlockChain, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, - engine consensus_engine.Engine, vmConfig vm.Config, options Options) (*BlockChainImpl, error) { + vmConfig vm.Config, options Options) (*BlockChainImpl, error) { bodyCache, _ := lru.New(bodyCacheLimit) bodyRLPCache, _ := lru.New(bodyCacheLimit) @@ -266,7 +267,6 @@ func newBlockChainWithOptions( blockAccumulatorCache: blockAccumulatorCache, leaderPubKeyFromCoinbase: leaderPubKeyFromCoinbase, blockchainPruner: newBlockchainPruner(db), - engine: engine, vmConfig: vmConfig, badBlocks: badBlocks, pendingSlashes: slash.Records{}, @@ -275,7 +275,7 @@ func newBlockChainWithOptions( } var err error - bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.getProcInterrupt) + bc.hc, err = NewHeaderChain(db, chainConfig, bc.getProcInterrupt) if err != nil { return nil, err } @@ -294,8 +294,8 @@ func newBlockChainWithOptions( beaconChain = bc } - bc.SetValidator(NewBlockValidator(chainConfig, bc, engine)) - bc.SetProcessor(NewStateProcessor(chainConfig, bc, beaconChain, engine)) + bc.SetValidator(NewBlockValidator(chainConfig, bc)) + bc.SetProcessor(NewStateProcessor(chainConfig, bc, beaconChain)) // Take ownership of this particular state go bc.update() @@ -353,7 +353,7 @@ func VerifyCrossLink(blockchain BlockChain, cl types.CrossLink) error { if blockchain.ShardID() != shard.BeaconChainShardID { return errors.New("[VerifyCrossLink] Shard chains should not verify cross links") } - engine := blockchain.Engine() + engine := chain.Engine() if err := engine.VerifyCrossLink(blockchain, cl); err != nil { return errors.Wrap(err, "[VerifyCrossLink]") @@ -424,7 +424,7 @@ func (bc *BlockChainImpl) ValidateNewBlock(block *types.Block, beaconChain Block Msg("[ValidateNewBlock] Cannot validate header for the new block") return err } - if err := bc.Engine().VerifyVRF( + if err := chain.Engine().VerifyVRF( bc, block.Header(), ); err != nil { utils.Logger().Error(). @@ -436,7 +436,7 @@ func (bc *BlockChainImpl) ValidateNewBlock(block *types.Block, beaconChain Block "[ValidateNewBlock] Cannot verify vrf for the new block", ) } - err := bc.Engine().VerifyShardState(bc, beaconChain, block.Header()) + err := chain.Engine().VerifyShardState(bc, beaconChain, block.Header()) if err != nil { utils.Logger().Error(). Str("blockHash", block.Hash().Hex()). @@ -1577,7 +1577,7 @@ func (bc *BlockChainImpl) insertChain(chain types.Blocks, verifyHeaders bool) (i seals[i] = true } // Note that VerifyHeaders verifies headers in the chain in parallel - abort, results := bc.Engine().VerifyHeaders(bc, headers, seals) + abort, results := chain2.Engine().VerifyHeaders(bc, headers, seals) verifyHeadersResults = results defer close(abort) } @@ -2045,8 +2045,6 @@ func (bc *BlockChainImpl) GetHeaderByNumber(number uint64) *block.Header { func (bc *BlockChainImpl) Config() *params.ChainConfig { return bc.chainConfig } -func (bc *BlockChainImpl) Engine() consensus_engine.Engine { return bc.engine } - func (bc *BlockChainImpl) SubscribeRemovedLogsEvent(ch chan<- RemovedLogsEvent) event.Subscription { return bc.scope.Track(bc.rmLogsFeed.Subscribe(ch)) } diff --git a/core/epochchain.go b/core/epochchain.go index bcf00f5a89..661d34ee4b 100644 --- a/core/epochchain.go +++ b/core/epochchain.go @@ -8,8 +8,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/harmony-one/harmony/block" - "github.com/harmony-one/harmony/consensus/engine" - consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" @@ -29,7 +27,6 @@ type EpochChain struct { db ethdb.Database // Low level persistent database to store final content in. mu chan struct{} currentHeader atomic.Value // Current head of the blockchain. - engine consensus_engine.Engine vmConfig *vm.Config headerCache *lru.Cache // Cache for the most recent block headers @@ -46,7 +43,7 @@ func cache(size int) *lru.Cache { } func NewEpochChain(db ethdb.Database, chainConfig *params.ChainConfig, - engine consensus_engine.Engine, vmConfig vm.Config) (*EpochChain, error) { + vmConfig vm.Config) (*EpochChain, error) { hash := rawdb.ReadCanonicalHash(db, 0) genesisBlock := rawdb.ReadBlock(db, hash, 0) @@ -60,7 +57,6 @@ func NewEpochChain(db ethdb.Database, chainConfig *params.ChainConfig, db: db, mu: make(chan struct{}, 1), currentHeader: atomic.Value{}, - engine: engine, vmConfig: &vmConfig, headerCache: cache(headerCacheLimit), @@ -242,10 +238,6 @@ func (bc *EpochChain) Config() *params.ChainConfig { return bc.chainConfig } -func (bc *EpochChain) Engine() engine.Engine { - return bc.engine -} - func (bc *EpochChain) ReadShardState(epoch *big.Int) (*shard.State, error) { cacheKey := string(epoch.Bytes()) if cached, ok := bc.shardStateCache.Get(cacheKey); ok { diff --git a/core/epochchain_test.go b/core/epochchain_test.go index eec7bd6280..7a0d75423c 100644 --- a/core/epochchain_test.go +++ b/core/epochchain_test.go @@ -15,7 +15,7 @@ func TestGenesisBlock(t *testing.T) { err := (&core.GenesisInitializer{NetworkType: nodeconfig.Mainnet}).InitChainDB(db, 0) require.NoError(t, err) - chain, err := core.NewEpochChain(db, nil, nil, vm.Config{}) + chain, err := core.NewEpochChain(db, nil, vm.Config{}) require.NoError(t, err) header := chain.GetHeaderByNumber(0) diff --git a/core/evm.go b/core/evm.go index f395ab042f..e11726a569 100644 --- a/core/evm.go +++ b/core/evm.go @@ -26,7 +26,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/block" - consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/internal/params" @@ -39,9 +38,6 @@ import ( // ChainContext supports retrieving headers and consensus parameters from the // current blockchain to be used during transaction processing. type ChainContext interface { - // Engine retrieves the chain's consensus engine. - Engine() consensus_engine.Engine - // GetHeader returns the hash corresponding to their hash. GetHeader(common.Hash, uint64) *block.Header diff --git a/core/evm_test.go b/core/evm_test.go index 962dedd307..ed6821ff58 100644 --- a/core/evm_test.go +++ b/core/evm_test.go @@ -21,7 +21,6 @@ import ( "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/crypto/hash" - chain2 "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/numeric" staking "github.com/harmony-one/harmony/staking/types" @@ -41,12 +40,11 @@ func getTestEnvironment(testBankKey ecdsa.PrivateKey) (*BlockChainImpl, *state.D Alloc: GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, ShardID: 0, } - engine = chain2.NewEngine() ) genesis := gspec.MustCommit(database) // fake blockchain - chain, _ := NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, engine, vm.Config{}) + chain, _ := NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, vm.Config{}) db, _ := chain.StateAt(genesis.Root()) // make a fake block header (use epoch 1 so that locked tokens can be tested) diff --git a/core/headerchain.go b/core/headerchain.go index a902d5a124..72cdaa490b 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -32,7 +32,6 @@ import ( lru "github.com/hashicorp/golang-lru" "github.com/harmony-one/harmony/block" - consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/utils" @@ -66,8 +65,7 @@ type HeaderChain struct { procInterrupt func() bool - rand *mrand.Rand - engine consensus_engine.Engine + rand *mrand.Rand } // NewHeaderChain creates a new HeaderChain structure. @@ -75,7 +73,7 @@ type HeaderChain struct { // getValidator should return the parent's validator // procInterrupt points to the parent's interrupt semaphore // wg points to the parent's shutdown wait group -func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine consensus_engine.Engine, procInterrupt func() bool) (*HeaderChain, error) { +func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, procInterrupt func() bool) (*HeaderChain, error) { headerCache, _ := lru.New(headerCacheLimit) tdCache, _ := lru.New(tdCacheLimit) numberCache, _ := lru.New(numberCacheLimit) @@ -96,7 +94,6 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine c canonicalCache: canonicalHash, procInterrupt: procInterrupt, rand: mrand.New(mrand.NewSource(seed.Int64())), - engine: engine, } hc.genesisHeader = hc.GetHeaderByNumber(0) @@ -548,9 +545,6 @@ func (hc *HeaderChain) SetGenesis(head *block.Header) { // Config retrieves the header chain's chain configuration. func (hc *HeaderChain) Config() *params.ChainConfig { return hc.config } -// Engine retrieves the header chain's consensus engine. -func (hc *HeaderChain) Engine() consensus_engine.Engine { return hc.engine } - // GetBlock implements consensus.ChainReader, and returns nil for every input as // a header chain does not have blocks available for retrieval. func (hc *HeaderChain) GetBlock(hash common.Hash, number uint64) *types.Block { diff --git a/core/state_processor.go b/core/state_processor.go index 23c7e35389..5477130c8c 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -20,13 +20,13 @@ import ( "math/big" "time" + "github.com/harmony-one/harmony/internal/chain" lru "github.com/hashicorp/golang-lru" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/block" - consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/consensus/reward" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" @@ -50,14 +50,13 @@ const ( // // StateProcessor implements Processor. type StateProcessor struct { - config *params.ChainConfig // Chain configuration options - bc BlockChain // Canonical blockchain - beacon BlockChain // Beacon chain - engine consensus_engine.Engine // Consensus engine used for block rewards - resultCache *lru.Cache // Cache for result after a certain block is processed + config *params.ChainConfig // Chain configuration options + bc BlockChain // Canonical blockchain + beacon BlockChain // Beacon chain + resultCache *lru.Cache // Cache for result after a certain block is processed } -// this structure is cached, and each individual element is returned +// ProcessorResult structure is cached, and each individual element is returned type ProcessorResult struct { Receipts types.Receipts CxReceipts types.CXReceipts @@ -70,7 +69,7 @@ type ProcessorResult struct { // NewStateProcessor initialises a new StateProcessor. func NewStateProcessor( - config *params.ChainConfig, bc BlockChain, beacon BlockChain, engine consensus_engine.Engine, + config *params.ChainConfig, bc BlockChain, beacon BlockChain, ) *StateProcessor { if bc == nil { panic("bc is nil") @@ -83,7 +82,6 @@ func NewStateProcessor( config: config, bc: bc, beacon: beacon, - engine: engine, resultCache: resultCache, } } @@ -195,7 +193,7 @@ func (p *StateProcessor) Process( // Block processing don't need to block on reward computation as in block proposal sigsReady <- true }() - _, payout, err := p.engine.Finalize( + _, payout, err := chain.Engine().Finalize( p.bc, p.beacon, header, statedb, block.Transactions(), diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index fa3af1c972..c3c3430d59 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -28,7 +28,6 @@ import ( "time" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/harmony-one/harmony/crypto/bls" "github.com/ethereum/go-ethereum/common" @@ -41,7 +40,6 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/crypto/hash" - chain2 "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/numeric" staking "github.com/harmony-one/harmony/staking/types" @@ -160,8 +158,7 @@ func createBlockChain() *BlockChainImpl { database := rawdb.NewMemoryDatabase() genesis := gspec.MustCommit(database) _ = genesis - engine := chain2.NewEngine() - blockchain, _ := NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, engine, vm.Config{}) + blockchain, _ := NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, vm.Config{}) return blockchain } diff --git a/hmy/downloader/adapter.go b/hmy/downloader/adapter.go index c8758b506d..3d8f6ac979 100644 --- a/hmy/downloader/adapter.go +++ b/hmy/downloader/adapter.go @@ -25,7 +25,6 @@ type syncProtocol interface { type blockChain interface { engine.ChainReader - Engine() engine.Engine InsertChain(chain types.Blocks, verifyHeaders bool) (int, error) WriteCommitSig(blockNum uint64, lastCommits []byte) error diff --git a/hmy/downloader/downloader.go b/hmy/downloader/downloader.go index 01ec242abb..b7d597c6bd 100644 --- a/hmy/downloader/downloader.go +++ b/hmy/downloader/downloader.go @@ -308,10 +308,10 @@ func verifyAndInsertBlock(bc blockChain, block *types.Block, nextBlocks ...*type } } - if err := bc.Engine().VerifyHeaderSignature(bc, block.Header(), sigBytes, bitmap); err != nil { + if err := chain.Engine().VerifyHeaderSignature(bc, block.Header(), sigBytes, bitmap); err != nil { return &sigVerifyErr{err} } - if err := bc.Engine().VerifyHeader(bc, block.Header(), true); err != nil { + if err := chain.Engine().VerifyHeader(bc, block.Header(), true); err != nil { return errors.Wrap(err, "[VerifyHeader]") } if _, err := bc.InsertChain(types.Blocks{block}, false); err != nil { diff --git a/hmy/tracer.go b/hmy/tracer.go index 699f784297..cdfa138aa6 100644 --- a/hmy/tracer.go +++ b/hmy/tracer.go @@ -39,6 +39,7 @@ import ( "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy/tracers" + "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/internal/utils" ) @@ -347,7 +348,7 @@ func (hmy *Harmony) TraceChain(ctx context.Context, start, end *types.Block, con // same as TraceBlock, but only use 1 thread func (hmy *Harmony) traceBlockNoThread(ctx context.Context, block *types.Block, config *TraceConfig) ([]*TxTraceResult, error) { // Create the parent state database - if err := hmy.BlockChain.Engine().VerifyHeader(hmy.BlockChain, block.Header(), true); err != nil { + if err := chain.Engine().VerifyHeader(hmy.BlockChain, block.Header(), true); err != nil { return nil, err } parent := hmy.BlockChain.GetBlock(block.ParentHash(), block.NumberU64()-1) @@ -422,7 +423,7 @@ func (hmy *Harmony) TraceBlock(ctx context.Context, block *types.Block, config * return hmy.traceBlockNoThread(ctx, block, config) } // Create the parent state database - if err := hmy.BlockChain.Engine().VerifyHeader(hmy.BlockChain, block.Header(), true); err != nil { + if err := chain.Engine().VerifyHeader(hmy.BlockChain, block.Header(), true); err != nil { return nil, err } parent := hmy.BlockChain.GetBlock(block.ParentHash(), block.NumberU64()-1) @@ -523,7 +524,7 @@ func (hmy *Harmony) standardTraceBlockToFile(ctx context.Context, block *types.B } } // Create the parent state database - if err := hmy.BlockChain.Engine().VerifyHeader(hmy.BlockChain, block.Header(), true); err != nil { + if err := chain.Engine().VerifyHeader(hmy.BlockChain, block.Header(), true); err != nil { return nil, err } parent := hmy.BlockChain.GetBlock(block.ParentHash(), block.NumberU64()-1) diff --git a/internal/chain/engine.go b/internal/chain/engine.go index 6e018dccfd..4337c2932a 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -6,6 +6,7 @@ import ( "sort" "time" + "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/internal/params" bls2 "github.com/harmony-one/bls/ffi/go/bls" @@ -16,7 +17,6 @@ import ( "github.com/pkg/errors" "github.com/harmony-one/harmony/block" - "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/consensus/reward" "github.com/harmony-one/harmony/consensus/signature" @@ -45,6 +45,8 @@ type engineImpl struct { verifiedSigCache *lru.Cache // verifiedSigKey -> struct{}{} } +var internal = NewEngine() + // NewEngine creates Engine with some cache func NewEngine() *engineImpl { sigCache, _ := lru.New(verifiedSigCache) @@ -55,6 +57,10 @@ func NewEngine() *engineImpl { } } +func Engine() *engineImpl { + return internal +} + // VerifyHeader checks whether a header conforms to the consensus rules of the bft engine. // Note that each block header contains the bls signature of the parent block func (e *engineImpl) VerifyHeader(chain engine.ChainReader, header *block.Header, seal bool) error { diff --git a/internal/shardchain/shardchains.go b/internal/shardchain/shardchains.go index 1be2c68414..a0f0e40ea3 100644 --- a/internal/shardchain/shardchains.go +++ b/internal/shardchain/shardchains.go @@ -13,7 +13,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" - "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/vm" @@ -40,7 +39,6 @@ type Collection interface { type CollectionImpl struct { dbFactory DBFactory dbInit DBInitializer - engine engine.Engine mtx sync.Mutex pool map[uint32]core.BlockChain disableCache map[uint32]bool @@ -56,14 +54,13 @@ type CollectionImpl struct { // the factory is brand new (empty). func NewCollection( harmonyconfig *harmonyconfig.HarmonyConfig, - dbFactory DBFactory, dbInit DBInitializer, engine engine.Engine, + dbFactory DBFactory, dbInit DBInitializer, chainConfig *params.ChainConfig, ) *CollectionImpl { return &CollectionImpl{ harmonyconfig: harmonyconfig, dbFactory: dbFactory, dbInit: dbInit, - engine: engine, pool: make(map[uint32]core.BlockChain), disableCache: make(map[uint32]bool), chainConfig: chainConfig, @@ -128,7 +125,7 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c } var bc core.BlockChain if opts.EpochChain { - bc, err = core.NewEpochChain(db, &chainConfig, sc.engine, vm.Config{}) + bc, err = core.NewEpochChain(db, &chainConfig, vm.Config{}) } else { stateCache, err := initStateCache(db, sc, shardID) if err != nil { @@ -136,7 +133,7 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c } if shardID == shard.BeaconChainShardID { bc, err = core.NewBlockChainWithOptions( - db, stateCache, bc, cacheConfig, &chainConfig, sc.engine, vm.Config{}, opts, + db, stateCache, bc, cacheConfig, &chainConfig, vm.Config{}, opts, ) } else { beacon, ok := sc.pool[shard.BeaconChainShardID] @@ -145,7 +142,7 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c } bc, err = core.NewBlockChainWithOptions( - db, stateCache, beacon, cacheConfig, &chainConfig, sc.engine, vm.Config{}, opts, + db, stateCache, beacon, cacheConfig, &chainConfig, vm.Config{}, opts, ) } } diff --git a/node/node.go b/node/node.go index 13988e2ad3..360a885e99 100644 --- a/node/node.go +++ b/node/node.go @@ -12,7 +12,6 @@ import ( "sync" "time" - "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/internal/registry" "github.com/harmony-one/harmony/internal/shardchain/tikv_manage" "github.com/harmony-one/harmony/internal/tikv" @@ -1015,7 +1014,6 @@ func (node *Node) GetSyncID() [SyncIDLength]byte { func New( host p2p.Host, consensusObj *consensus.Consensus, - engine engine.Engine, collection *shardchain.CollectionImpl, blacklist map[common.Address]struct{}, allowedTxs map[common.Address]core.AllowedTxData, @@ -1110,7 +1108,7 @@ func New( node.TxPool = core.NewTxPool(txPoolConfig, node.Blockchain().Config(), blockchain, node.TransactionErrorSink) node.CxPool = core.NewCxPool(core.CxPoolSize) - node.Worker = worker.New(node.Blockchain().Config(), blockchain, beaconChain, engine) + node.Worker = worker.New(node.Blockchain().Config(), blockchain, beaconChain) node.deciderCache, _ = lru.New(16) node.committeeCache, _ = lru.New(16) diff --git a/node/node_cross_link.go b/node/node_cross_link.go index d0f80545d5..0a919368d1 100644 --- a/node/node_cross_link.go +++ b/node/node_cross_link.go @@ -8,6 +8,7 @@ import ( ffi_bls "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" "github.com/pkg/errors" @@ -179,7 +180,7 @@ func (node *Node) VerifyCrossLink(cl types.CrossLink) error { if cl.ShardID() >= instance.NumShards() { return errors.New("[VerifyCrossLink] ShardID should less than NumShards") } - engine := node.Blockchain().Engine() + engine := chain.Engine() if err := engine.VerifyCrossLink(node.Blockchain(), cl); err != nil { return errors.Wrap(err, "[VerifyCrossLink]") diff --git a/node/node_handler_test.go b/node/node_handler_test.go index 25d966f469..b0d276d714 100644 --- a/node/node_handler_test.go +++ b/node/node_handler_test.go @@ -33,10 +33,9 @@ func TestAddNewBlock(t *testing.T) { if err != nil { t.Fatalf("newhost failure: %v", err) } - engine := chain.NewEngine() chainconfig := nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType().ChainConfig() collection := shardchain.NewCollection( - nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, engine, &chainconfig, + nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, &chainconfig, ) decider := quorum.NewDecider( quorum.SuperMajorityVote, shard.BeaconChainShardID, @@ -53,7 +52,7 @@ func TestAddNewBlock(t *testing.T) { t.Fatalf("Cannot craeate consensus: %v", err) } nodeconfig.SetNetworkType(nodeconfig.Devnet) - node := New(host, consensus, engine, collection, nil, nil, nil, nil, nil, reg) + node := New(host, consensus, collection, nil, nil, nil, nil, nil, reg) txs := make(map[common.Address]types.Transactions) stks := staking.StakingTransactions{} @@ -90,10 +89,9 @@ func TestVerifyNewBlock(t *testing.T) { if err != nil { t.Fatalf("newhost failure: %v", err) } - engine := chain.NewEngine() chainconfig := nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType().ChainConfig() collection := shardchain.NewCollection( - nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, engine, &chainconfig, + nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, &chainconfig, ) decider := quorum.NewDecider( quorum.SuperMajorityVote, shard.BeaconChainShardID, @@ -112,7 +110,7 @@ func TestVerifyNewBlock(t *testing.T) { archiveMode := make(map[uint32]bool) archiveMode[0] = true archiveMode[1] = false - node := New(host, consensus, engine, collection, nil, nil, nil, archiveMode, nil, reg) + node := New(host, consensus, collection, nil, nil, nil, archiveMode, nil, reg) txs := make(map[common.Address]types.Transactions) stks := staking.StakingTransactions{} @@ -146,10 +144,9 @@ func TestVerifyVRF(t *testing.T) { if err != nil { t.Fatalf("newhost failure: %v", err) } - engine := chain.NewEngine() chainconfig := nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType().ChainConfig() collection := shardchain.NewCollection( - nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, engine, &chainconfig, + nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, &chainconfig, ) blockchain, err := collection.ShardChain(shard.BeaconChainShardID) if err != nil { @@ -168,7 +165,7 @@ func TestVerifyVRF(t *testing.T) { archiveMode := make(map[uint32]bool) archiveMode[0] = true archiveMode[1] = false - node := New(host, consensus, engine, collection, nil, nil, nil, archiveMode, nil, reg) + node := New(host, consensus, collection, nil, nil, nil, archiveMode, nil, reg) txs := make(map[common.Address]types.Transactions) stks := staking.StakingTransactions{} @@ -205,7 +202,7 @@ func TestVerifyVRF(t *testing.T) { node.Blockchain().WriteShardStateBytes(node.Blockchain().ChainDb(), big.NewInt(1), node.Worker.GetCurrentHeader().ShardState()) node.Blockchain().Config().VRFEpoch = big.NewInt(0) - if err := node.Blockchain().Engine().VerifyVRF( + if err := chain.Engine().VerifyVRF( node.Blockchain(), block.Header(), ); err != nil { t.Error("New vrf is not verified successfully:", err) diff --git a/node/node_newblock_test.go b/node/node_newblock_test.go index 492175b1d2..5483e32c3b 100644 --- a/node/node_newblock_test.go +++ b/node/node_newblock_test.go @@ -10,7 +10,6 @@ import ( "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/crypto/bls" - "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/registry" "github.com/harmony-one/harmony/internal/shardchain" @@ -35,10 +34,9 @@ func TestFinalizeNewBlockAsync(t *testing.T) { t.Fatalf("newhost failure: %v", err) } var testDBFactory = &shardchain.MemDBFactory{} - engine := chain.NewEngine() chainconfig := nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType().ChainConfig() collection := shardchain.NewCollection( - nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, engine, &chainconfig, + nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, &chainconfig, ) blockchain, err := collection.ShardChain(shard.BeaconChainShardID) require.NoError(t, err) @@ -53,7 +51,7 @@ func TestFinalizeNewBlockAsync(t *testing.T) { t.Fatalf("Cannot craeate consensus: %v", err) } - node := New(host, consensus, engine, collection, nil, nil, nil, nil, nil, registry.New().SetBlockchain(blockchain)) + node := New(host, consensus, collection, nil, nil, nil, nil, nil, registry.New().SetBlockchain(blockchain)) node.Worker.UpdateCurrent() diff --git a/node/node_test.go b/node/node_test.go index 49ba5d164d..86bd2b10bd 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -8,7 +8,6 @@ import ( "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/crypto/bls" - "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/registry" "github.com/harmony-one/harmony/internal/shardchain" @@ -34,13 +33,12 @@ func TestNewNode(t *testing.T) { if err != nil { t.Fatalf("newhost failure: %v", err) } - engine := chain.NewEngine() decider := quorum.NewDecider( quorum.SuperMajorityVote, shard.BeaconChainShardID, ) chainconfig := nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType().ChainConfig() collection := shardchain.NewCollection( - nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, engine, &chainconfig, + nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, &chainconfig, ) blockchain, err := collection.ShardChain(shard.BeaconChainShardID) if err != nil { @@ -54,7 +52,7 @@ func TestNewNode(t *testing.T) { t.Fatalf("Cannot craeate consensus: %v", err) } - node := New(host, consensus, engine, collection, nil, nil, nil, nil, nil, reg) + node := New(host, consensus, collection, nil, nil, nil, nil, nil, reg) if node.Consensus == nil { t.Error("Consensus is not initialized for the node") } diff --git a/node/worker/worker.go b/node/worker/worker.go index 1a711bc193..f8279d3495 100644 --- a/node/worker/worker.go +++ b/node/worker/worker.go @@ -8,6 +8,7 @@ import ( "time" "github.com/harmony-one/harmony/consensus/reward" + "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/consensus" @@ -19,7 +20,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/block" blockfactory "github.com/harmony-one/harmony/block/factory" - consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" @@ -59,7 +59,6 @@ type Worker struct { chain core.BlockChain beacon core.BlockChain current *environment // An environment for current running cycle. - engine consensus_engine.Engine gasFloor uint64 gasCeil uint64 } @@ -558,7 +557,7 @@ func (w *Worker) FinalizeNewBlock( } }() - block, payout, err := w.engine.Finalize( + block, payout, err := chain.Engine().Finalize( w.chain, w.beacon, copyHeader, state, w.current.txs, w.current.receipts, @@ -574,14 +573,13 @@ func (w *Worker) FinalizeNewBlock( // New create a new worker object. func New( - config *params.ChainConfig, chain core.BlockChain, beacon core.BlockChain, engine consensus_engine.Engine, + config *params.ChainConfig, chain core.BlockChain, beacon core.BlockChain, ) *Worker { worker := &Worker{ config: config, factory: blockfactory.NewFactory(config), chain: chain, beacon: beacon, - engine: engine, } worker.gasFloor = 80000000 worker.gasCeil = 120000000 diff --git a/node/worker/worker_test.go b/node/worker/worker_test.go index e6ffdfddad..7bd25c8db8 100644 --- a/node/worker/worker_test.go +++ b/node/worker/worker_test.go @@ -16,7 +16,6 @@ import ( "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" - chain2 "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/internal/params" ) @@ -40,18 +39,17 @@ func TestNewWorker(t *testing.T) { Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, ShardID: 10, } - engine = chain2.NewEngine() ) genesis := gspec.MustCommit(database) _ = genesis - chain, err := core.NewBlockChain(database, state.NewDatabase(database), &core.BlockChainImpl{}, nil, gspec.Config, engine, vm.Config{}) + chain, err := core.NewBlockChain(database, state.NewDatabase(database), &core.BlockChainImpl{}, nil, gspec.Config, vm.Config{}) if err != nil { t.Error(err) } // Create a new worker - worker := New(params.TestChainConfig, chain, nil, engine) + worker := New(params.TestChainConfig, chain, nil) if worker.GetCurrentState().GetBalance(crypto.PubkeyToAddress(testBankKey.PublicKey)).Cmp(testBankFunds) != 0 { t.Error("Worker state is not setup correctly") @@ -68,14 +66,13 @@ func TestCommitTransactions(t *testing.T) { Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, ShardID: 0, } - engine = chain2.NewEngine() ) gspec.MustCommit(database) - chain, _ := core.NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, engine, vm.Config{}) + chain, _ := core.NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, vm.Config{}) // Create a new worker - worker := New(params.TestChainConfig, chain, nil, engine) + worker := New(params.TestChainConfig, chain, nil) // Generate a test tx baseNonce := worker.GetCurrentState().GetNonce(crypto.PubkeyToAddress(testBankKey.PublicKey)) diff --git a/test/chain/chain/chain_makers.go b/test/chain/chain/chain_makers.go index b4ca9ecd7c..d5c10a5d59 100644 --- a/test/chain/chain/chain_makers.go +++ b/test/chain/chain/chain_makers.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/block" blockfactory "github.com/harmony-one/harmony/block/factory" @@ -49,7 +50,6 @@ type BlockGen struct { receipts []*types.Receipt uncles []*block.Header config *params.ChainConfig - engine consensus_engine.Engine } // SetCoinbase sets the coinbase of the generated block. @@ -167,7 +167,7 @@ func (b *BlockGen) PrevBlock(index int) *types.Block { // a similar non-validating proof of work implementation. func GenerateChain( config *params.ChainConfig, parent *types.Block, - engine consensus_engine.Engine, db ethdb.Database, + db ethdb.Database, n int, gen func(int, *BlockGen), ) ([]*types.Block, []types.Receipts) { @@ -185,18 +185,17 @@ func GenerateChain( statedb: statedb, config: config, factory: factory, - engine: engine, } - b.header = makeHeader(chainreader, parent, statedb, b.engine, factory) + b.header = makeHeader(chainreader, parent, statedb, factory) // Execute any user modifications to the block if gen != nil { gen(i, b) } - if b.engine != nil { + if true { // Finalize and seal the block - block, _, err := b.engine.Finalize( + block, _, err := chain.Engine().Finalize( chainreader, nil, b.header, statedb, b.txs, b.receipts, nil, nil, nil, nil, nil, func() uint64 { return 0 }, ) if err != nil { @@ -228,7 +227,7 @@ func GenerateChain( return blocks, receipts } -func makeHeader(chain consensus_engine.ChainReader, parent *types.Block, state *state.DB, engine consensus_engine.Engine, factory blockfactory.Factory) *block.Header { +func makeHeader(chain consensus_engine.ChainReader, parent *types.Block, state *state.DB, factory blockfactory.Factory) *block.Header { var time *big.Int if parent.Time() == nil { time = big.NewInt(10) diff --git a/test/chain/main.go b/test/chain/main.go index e78b519356..6da9255692 100644 --- a/test/chain/main.go +++ b/test/chain/main.go @@ -94,7 +94,7 @@ func fundFaucetContract(chain core.BlockChain) { fmt.Println("--------- Funding addresses for Faucet Contract Call ---------") fmt.Println() - contractworker = pkgworker.New(params.TestChainConfig, chain, nil, chain.Engine()) + contractworker = pkgworker.New(params.TestChainConfig, chain, nil) nonce = contractworker.GetCurrentState().GetNonce(crypto.PubkeyToAddress(FaucetPriKey.PublicKey)) dataEnc = common.FromHex(FaucetContractBinary) ftx, _ := types.SignTx( @@ -206,7 +206,7 @@ func playFaucetContract(chain core.BlockChain) { func main() { genesis := gspec.MustCommit(database) - chain, _ := core.NewBlockChain(database, harmonyState.NewDatabase(database), nil, nil, gspec.Config, chain.Engine(), vm.Config{}) + chain, _ := core.NewBlockChain(database, harmonyState.NewDatabase(database), nil, nil, gspec.Config, vm.Config{}) txpool := core.NewTxPool(core.DefaultTxPoolConfig, chainConfig, chain, types.NewTransactionErrorSink()) backend := &testWorkerBackend{ @@ -223,7 +223,7 @@ func main() { //// Generate a small n-block chain and an uncle block for it n := 3 if n > 0 { - blocks, _ := chain2.GenerateChain(chainConfig, genesis, chain.Engine(), database, n, func(i int, gen *chain2.BlockGen) { + blocks, _ := chain2.GenerateChain(chainConfig, genesis, database, n, func(i int, gen *chain2.BlockGen) { gen.SetCoinbase(FaucetAddress) gen.SetShardID(0) gen.AddTx(pendingTxs[i].(*types.Transaction)) diff --git a/test/chain/reward/main.go b/test/chain/reward/main.go index 3848697b66..828c6fc7f9 100644 --- a/test/chain/reward/main.go +++ b/test/chain/reward/main.go @@ -22,7 +22,6 @@ import ( "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/crypto/hash" - "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/internal/common" protobuf "github.com/golang/protobuf/proto" @@ -108,8 +107,7 @@ func main() { database := rawdb.NewMemoryDatabase() genesis := gspec.MustCommit(database) _ = genesis - engine := chain.NewEngine() - bc, _ := core.NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, engine, vm.Config{}) + bc, _ := core.NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, vm.Config{}) statedb, _ := state.New(common2.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) msg := createValidator() statedb.AddBalance(msg.ValidatorAddress, new(big.Int).Mul(big.NewInt(5e18), big.NewInt(2000))) From 1a70d5eb66cdbf8a23791806b71a323eed320085 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Tue, 21 Mar 2023 18:34:07 -0300 Subject: [PATCH 377/420] Fix possible panic. --- core/blockchain_stub.go | 4 ---- core/epochchain.go | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/core/blockchain_stub.go b/core/blockchain_stub.go index f9e9111eae..f1c27f5743 100644 --- a/core/blockchain_stub.go +++ b/core/blockchain_stub.go @@ -141,10 +141,6 @@ func (a Stub) Config() *params.ChainConfig { return nil } -func (a Stub) Engine() engine.Engine { - return nil -} - func (a Stub) SubscribeRemovedLogsEvent(ch chan<- RemovedLogsEvent) event.Subscription { return nil } diff --git a/core/epochchain.go b/core/epochchain.go index 661d34ee4b..507142f392 100644 --- a/core/epochchain.go +++ b/core/epochchain.go @@ -128,7 +128,7 @@ func (bc *EpochChain) InsertChain(blocks types.Blocks, _ bool) (int, error) { } // Signature validation. - err = bc.Engine().VerifyHeaderSignature(bc, block.Header(), sig, bitmap) + err = chain.Engine().VerifyHeaderSignature(bc, block.Header(), sig, bitmap) if err != nil { return i, errors.Wrap(err, "failed signature validation") } From 15885f4c9b9263746827172b4f4f56d0926d18e2 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 22 Mar 2023 01:25:20 -0300 Subject: [PATCH 378/420] Clean up code. --- node/worker/worker.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/node/worker/worker.go b/node/worker/worker.go index f8279d3495..d2a8f407a4 100644 --- a/node/worker/worker.go +++ b/node/worker/worker.go @@ -311,7 +311,7 @@ func (w *Worker) UpdateCurrent() error { Time(big.NewInt(timestamp)). ShardID(w.chain.ShardID()). Header() - return w.makeCurrent(parent, header) + return w.makeCurrent(parent.Header(), header) } // GetCurrentHeader returns the current header to propose @@ -320,7 +320,7 @@ func (w *Worker) GetCurrentHeader() *block.Header { } // makeCurrent creates a new environment for the current cycle. -func (w *Worker) makeCurrent(parent *types.Block, header *block.Header) error { +func (w *Worker) makeCurrent(parent *block.Header, header *block.Header) error { state, err := w.chain.StateAt(parent.Root()) if err != nil { return err @@ -380,16 +380,6 @@ func (w *Worker) GetCurrentReceipts() []*types.Receipt { return w.current.receipts } -// OutgoingReceipts get the receipts generated starting from the last state. -func (w *Worker) OutgoingReceipts() []*types.CXReceipt { - return w.current.outcxs -} - -// IncomingReceipts get incoming receipts in destination shard that is received from source shard -func (w *Worker) IncomingReceipts() []*types.CXReceiptsProof { - return w.current.incxs -} - // CollectVerifiedSlashes sets w.current.slashes only to those that // past verification func (w *Worker) CollectVerifiedSlashes() error { @@ -596,7 +586,7 @@ func New( Time(big.NewInt(timestamp)). ShardID(worker.chain.ShardID()). Header() - worker.makeCurrent(parent, header) + worker.makeCurrent(parent.Header(), header) return worker } From 5e1878aedca3989dc0f34161dae1833e43ca6a76 Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 22 Mar 2023 01:48:12 -0300 Subject: [PATCH 379/420] Network type. --- cmd/harmony/main.go | 3 +-- core/genesis_initializer.go | 9 ++++----- internal/shardchain/dbinit.go | 7 +++++-- internal/shardchain/shardchains.go | 12 ++++++------ 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index d13ba266e9..afffa1f3c6 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -700,9 +700,8 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi chainDBFactory = &shardchain.LDBFactory{RootDir: nodeConfig.DBDir} } - chainConfig := nodeConfig.GetNetworkType().ChainConfig() collection := shardchain.NewCollection( - &hc, chainDBFactory, &core.GenesisInitializer{NetworkType: nodeConfig.GetNetworkType()}, &chainConfig, + &hc, chainDBFactory, &core.GenesisInitializer{}, nodeConfig.GetNetworkType(), ) for shardID, archival := range nodeConfig.ArchiveModes() { if archival { diff --git a/core/genesis_initializer.go b/core/genesis_initializer.go index 04c6c85ed6..87ddd5fff3 100644 --- a/core/genesis_initializer.go +++ b/core/genesis_initializer.go @@ -13,11 +13,10 @@ import ( // GenesisInitializer is a shardchain.DBInitializer adapter. type GenesisInitializer struct { - NetworkType nodeconfig.NetworkType } // InitChainDB sets up a new genesis block in the database for the given shard. -func (gi *GenesisInitializer) InitChainDB(db ethdb.Database, shardID uint32) error { +func (gi *GenesisInitializer) InitChainDB(db ethdb.Database, networkType nodeconfig.NetworkType, shardID uint32) error { shardState, _ := committee.WithStakingEnabled.Compute( big.NewInt(GenesisEpoch), nil, ) @@ -34,14 +33,14 @@ func (gi *GenesisInitializer) InitChainDB(db ethdb.Database, shardID uint32) err } shardState = &shard.State{Shards: []shard.Committee{*subComm}} } - gi.setupGenesisBlock(db, shardID, shardState) + gi.setupGenesisBlock(db, shardID, shardState, networkType) return nil } // SetupGenesisBlock sets up a genesis blockchain. -func (gi *GenesisInitializer) setupGenesisBlock(db ethdb.Database, shardID uint32, myShardState *shard.State) { +func (gi *GenesisInitializer) setupGenesisBlock(db ethdb.Database, shardID uint32, myShardState *shard.State, networkType nodeconfig.NetworkType) { utils.Logger().Info().Interface("shardID", shardID).Msg("setting up a brand new chain database") - gspec := NewGenesisSpec(gi.NetworkType, shardID) + gspec := NewGenesisSpec(networkType, shardID) gspec.ShardStateHash = myShardState.Hash() gspec.ShardState = *myShardState.DeepCopy() // Store genesis block into db. diff --git a/internal/shardchain/dbinit.go b/internal/shardchain/dbinit.go index 4987a78862..b7b5d9e38a 100644 --- a/internal/shardchain/dbinit.go +++ b/internal/shardchain/dbinit.go @@ -1,8 +1,11 @@ package shardchain -import "github.com/ethereum/go-ethereum/ethdb" +import ( + "github.com/ethereum/go-ethereum/ethdb" + nodeconfig "github.com/harmony-one/harmony/internal/configs/node" +) // DBInitializer initializes a newly created chain database. type DBInitializer interface { - InitChainDB(db ethdb.Database, shardID uint32) error + InitChainDB(db ethdb.Database, networkType nodeconfig.NetworkType, shardID uint32) error } diff --git a/internal/shardchain/shardchains.go b/internal/shardchain/shardchains.go index a0f0e40ea3..957ba9150b 100644 --- a/internal/shardchain/shardchains.go +++ b/internal/shardchain/shardchains.go @@ -7,6 +7,7 @@ import ( "github.com/harmony-one/harmony/core/state" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" + nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/shardchain/tikv_manage" "github.com/harmony-one/harmony/shard" @@ -16,7 +17,6 @@ import ( "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/vm" - "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/utils" "github.com/pkg/errors" ) @@ -42,7 +42,7 @@ type CollectionImpl struct { mtx sync.Mutex pool map[uint32]core.BlockChain disableCache map[uint32]bool - chainConfig *params.ChainConfig + networkType nodeconfig.NetworkType harmonyconfig *harmonyconfig.HarmonyConfig } @@ -55,7 +55,7 @@ type CollectionImpl struct { func NewCollection( harmonyconfig *harmonyconfig.HarmonyConfig, dbFactory DBFactory, dbInit DBInitializer, - chainConfig *params.ChainConfig, + network nodeconfig.NetworkType, ) *CollectionImpl { return &CollectionImpl{ harmonyconfig: harmonyconfig, @@ -63,7 +63,7 @@ func NewCollection( dbInit: dbInit, pool: make(map[uint32]core.BlockChain), disableCache: make(map[uint32]bool), - chainConfig: chainConfig, + networkType: network, } } @@ -92,7 +92,7 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c utils.Logger().Info(). Uint32("shardID", shardID). Msg("initializing a new chain database") - if err := sc.dbInit.InitChainDB(db, shardID); err != nil { + if err := sc.dbInit.InitChainDB(db, sc.networkType, shardID); err != nil { return nil, errors.Wrapf(err, "cannot initialize a new chain database") } } @@ -113,7 +113,7 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c } } - chainConfig := *sc.chainConfig + chainConfig := sc.networkType.ChainConfig() if shardID == shard.BeaconChainShardID { // For beacon chain inside a shard chain, need to reset the eth chainID to shard 0's eth chainID in the config From 597ba2d6f1ed54ff599b9d9b8940c7285ab1277a Mon Sep 17 00:00:00 2001 From: frozen <355847+Frozen@users.noreply.github.com> Date: Wed, 22 Mar 2023 03:35:25 -0300 Subject: [PATCH 380/420] Fix tests. --- consensus/engine/consensus_engine.go | 2 +- core/blockchain.go | 2 ++ core/blockchain_impl.go | 24 +++++++++++++----------- core/blockchain_stub.go | 4 ++++ core/epochchain.go | 7 ++++++- core/epochchain_test.go | 2 +- core/evm_test.go | 3 ++- core/tx_pool_test.go | 3 ++- hmy/downloader/adapter.go | 1 + hmy/downloader/downloader.go | 4 ++-- internal/chain/engine.go | 6 +++--- internal/shardchain/shardchains.go | 5 +++-- node/node_handler_test.go | 9 +++------ node/node_newblock_test.go | 3 +-- node/node_test.go | 3 +-- node/worker/worker_test.go | 5 +++-- test/chain/main.go | 2 +- test/chain/reward/main.go | 4 +++- 18 files changed, 52 insertions(+), 37 deletions(-) diff --git a/consensus/engine/consensus_engine.go b/consensus/engine/consensus_engine.go index 3e7cc3b348..0879a5f546 100644 --- a/consensus/engine/consensus_engine.go +++ b/consensus/engine/consensus_engine.go @@ -79,7 +79,7 @@ type Engine interface { // via the VerifySeal method. VerifyHeader(chain ChainReader, header *block.Header, seal bool) error - // Similiar to VerifyHeader, which is only for verifying the block headers of one's own chain, this verification + // VerifyHeaderSignature similar to VerifyHeader, which is only for verifying the block headers of one's own chain, this verification // is used for verifying "incoming" block header against commit signature and bitmap sent from the other chain cross-shard via libp2p. // i.e. this header verification api is more flexible since the caller specifies which commit signature and bitmap to use // for verifying the block header, which is necessary for cross-shard block header verification. Example of such is cross-shard transaction. diff --git a/core/blockchain.go b/core/blockchain.go index a28df6bab1..fda4831655 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -126,6 +126,8 @@ type BlockChain interface { GetHeaderByNumber(number uint64) *block.Header // Config retrieves the blockchain's chain configuration. Config() *params.ChainConfig + // Engine retrieves the blockchain's consensus engine. + Engine() engine.Engine // SubscribeRemovedLogsEvent registers a subscription of RemovedLogsEvent. SubscribeRemovedLogsEvent(ch chan<- RemovedLogsEvent) event.Subscription // SubscribeTraceEvent registers a subscription of ChainEvent. diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 5ed1224207..515c4099fa 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -49,8 +49,6 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/crypto/bls" - "github.com/harmony-one/harmony/internal/chain" - chain2 "github.com/harmony-one/harmony/internal/chain" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/tikv" @@ -192,6 +190,7 @@ type BlockChainImpl struct { // procInterrupt must be atomically called procInterrupt int32 // interrupt signaler for block processing + engine consensus_engine.Engine processor Processor // block processor interface validator Validator // block and state validator interface vmConfig vm.Config @@ -205,9 +204,9 @@ type BlockChainImpl struct { // NewBlockChainWithOptions same as NewBlockChain but can accept additional behaviour options. func NewBlockChainWithOptions( db ethdb.Database, stateCache state.Database, beaconChain BlockChain, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, - vmConfig vm.Config, options Options, + engine consensus_engine.Engine, vmConfig vm.Config, options Options, ) (*BlockChainImpl, error) { - return newBlockChainWithOptions(db, stateCache, beaconChain, cacheConfig, chainConfig, vmConfig, options) + return newBlockChainWithOptions(db, stateCache, beaconChain, cacheConfig, chainConfig, engine, vmConfig, options) } // NewBlockChain returns a fully initialised block chain using information @@ -215,15 +214,15 @@ func NewBlockChainWithOptions( // Processor. func NewBlockChain( db ethdb.Database, stateCache state.Database, beaconChain BlockChain, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, - vmConfig vm.Config, + engine consensus_engine.Engine, vmConfig vm.Config, ) (*BlockChainImpl, error) { - return newBlockChainWithOptions(db, stateCache, beaconChain, cacheConfig, chainConfig, vmConfig, Options{}) + return newBlockChainWithOptions(db, stateCache, beaconChain, cacheConfig, chainConfig, engine, vmConfig, Options{}) } func newBlockChainWithOptions( db ethdb.Database, stateCache state.Database, beaconChain BlockChain, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, - vmConfig vm.Config, options Options) (*BlockChainImpl, error) { + engine consensus_engine.Engine, vmConfig vm.Config, options Options) (*BlockChainImpl, error) { bodyCache, _ := lru.New(bodyCacheLimit) bodyRLPCache, _ := lru.New(bodyCacheLimit) @@ -267,6 +266,7 @@ func newBlockChainWithOptions( blockAccumulatorCache: blockAccumulatorCache, leaderPubKeyFromCoinbase: leaderPubKeyFromCoinbase, blockchainPruner: newBlockchainPruner(db), + engine: engine, vmConfig: vmConfig, badBlocks: badBlocks, pendingSlashes: slash.Records{}, @@ -353,7 +353,7 @@ func VerifyCrossLink(blockchain BlockChain, cl types.CrossLink) error { if blockchain.ShardID() != shard.BeaconChainShardID { return errors.New("[VerifyCrossLink] Shard chains should not verify cross links") } - engine := chain.Engine() + engine := blockchain.Engine() if err := engine.VerifyCrossLink(blockchain, cl); err != nil { return errors.Wrap(err, "[VerifyCrossLink]") @@ -424,7 +424,7 @@ func (bc *BlockChainImpl) ValidateNewBlock(block *types.Block, beaconChain Block Msg("[ValidateNewBlock] Cannot validate header for the new block") return err } - if err := chain.Engine().VerifyVRF( + if err := bc.Engine().VerifyVRF( bc, block.Header(), ); err != nil { utils.Logger().Error(). @@ -436,7 +436,7 @@ func (bc *BlockChainImpl) ValidateNewBlock(block *types.Block, beaconChain Block "[ValidateNewBlock] Cannot verify vrf for the new block", ) } - err := chain.Engine().VerifyShardState(bc, beaconChain, block.Header()) + err := bc.Engine().VerifyShardState(bc, beaconChain, block.Header()) if err != nil { utils.Logger().Error(). Str("blockHash", block.Hash().Hex()). @@ -1577,7 +1577,7 @@ func (bc *BlockChainImpl) insertChain(chain types.Blocks, verifyHeaders bool) (i seals[i] = true } // Note that VerifyHeaders verifies headers in the chain in parallel - abort, results := chain2.Engine().VerifyHeaders(bc, headers, seals) + abort, results := bc.Engine().VerifyHeaders(bc, headers, seals) verifyHeadersResults = results defer close(abort) } @@ -2045,6 +2045,8 @@ func (bc *BlockChainImpl) GetHeaderByNumber(number uint64) *block.Header { func (bc *BlockChainImpl) Config() *params.ChainConfig { return bc.chainConfig } +func (bc *BlockChainImpl) Engine() consensus_engine.Engine { return bc.engine } + func (bc *BlockChainImpl) SubscribeRemovedLogsEvent(ch chan<- RemovedLogsEvent) event.Subscription { return bc.scope.Track(bc.rmLogsFeed.Subscribe(ch)) } diff --git a/core/blockchain_stub.go b/core/blockchain_stub.go index f1c27f5743..f9e9111eae 100644 --- a/core/blockchain_stub.go +++ b/core/blockchain_stub.go @@ -141,6 +141,10 @@ func (a Stub) Config() *params.ChainConfig { return nil } +func (a Stub) Engine() engine.Engine { + return nil +} + func (a Stub) SubscribeRemovedLogsEvent(ch chan<- RemovedLogsEvent) event.Subscription { return nil } diff --git a/core/epochchain.go b/core/epochchain.go index 507142f392..ede5ec5a79 100644 --- a/core/epochchain.go +++ b/core/epochchain.go @@ -8,6 +8,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/harmony-one/harmony/block" + "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" @@ -128,7 +129,7 @@ func (bc *EpochChain) InsertChain(blocks types.Blocks, _ bool) (int, error) { } // Signature validation. - err = chain.Engine().VerifyHeaderSignature(bc, block.Header(), sig, bitmap) + err = bc.Engine().VerifyHeaderSignature(bc, block.Header(), sig, bitmap) if err != nil { return i, errors.Wrap(err, "failed signature validation") } @@ -238,6 +239,10 @@ func (bc *EpochChain) Config() *params.ChainConfig { return bc.chainConfig } +func (bc *EpochChain) Engine() engine.Engine { + return chain.Engine() +} + func (bc *EpochChain) ReadShardState(epoch *big.Int) (*shard.State, error) { cacheKey := string(epoch.Bytes()) if cached, ok := bc.shardStateCache.Get(cacheKey); ok { diff --git a/core/epochchain_test.go b/core/epochchain_test.go index 7a0d75423c..91c85bfdb3 100644 --- a/core/epochchain_test.go +++ b/core/epochchain_test.go @@ -12,7 +12,7 @@ import ( func TestGenesisBlock(t *testing.T) { db := rawdb.NewMemoryDatabase() - err := (&core.GenesisInitializer{NetworkType: nodeconfig.Mainnet}).InitChainDB(db, 0) + err := (&core.GenesisInitializer{}).InitChainDB(db, nodeconfig.Mainnet, 0) require.NoError(t, err) chain, err := core.NewEpochChain(db, nil, vm.Config{}) diff --git a/core/evm_test.go b/core/evm_test.go index ed6821ff58..3201ee116b 100644 --- a/core/evm_test.go +++ b/core/evm_test.go @@ -21,6 +21,7 @@ import ( "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/crypto/hash" + "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/numeric" staking "github.com/harmony-one/harmony/staking/types" @@ -44,7 +45,7 @@ func getTestEnvironment(testBankKey ecdsa.PrivateKey) (*BlockChainImpl, *state.D genesis := gspec.MustCommit(database) // fake blockchain - chain, _ := NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, vm.Config{}) + chain, _ := NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, chain.Engine(), vm.Config{}) db, _ := chain.StateAt(genesis.Root()) // make a fake block header (use epoch 1 so that locked tokens can be tested) diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index c3c3430d59..d6b181e182 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/harmony-one/harmony/crypto/bls" + "github.com/harmony-one/harmony/internal/chain" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" @@ -158,7 +159,7 @@ func createBlockChain() *BlockChainImpl { database := rawdb.NewMemoryDatabase() genesis := gspec.MustCommit(database) _ = genesis - blockchain, _ := NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, vm.Config{}) + blockchain, _ := NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, chain.Engine(), vm.Config{}) return blockchain } diff --git a/hmy/downloader/adapter.go b/hmy/downloader/adapter.go index 3d8f6ac979..c8758b506d 100644 --- a/hmy/downloader/adapter.go +++ b/hmy/downloader/adapter.go @@ -25,6 +25,7 @@ type syncProtocol interface { type blockChain interface { engine.ChainReader + Engine() engine.Engine InsertChain(chain types.Blocks, verifyHeaders bool) (int, error) WriteCommitSig(blockNum uint64, lastCommits []byte) error diff --git a/hmy/downloader/downloader.go b/hmy/downloader/downloader.go index b7d597c6bd..01ec242abb 100644 --- a/hmy/downloader/downloader.go +++ b/hmy/downloader/downloader.go @@ -308,10 +308,10 @@ func verifyAndInsertBlock(bc blockChain, block *types.Block, nextBlocks ...*type } } - if err := chain.Engine().VerifyHeaderSignature(bc, block.Header(), sigBytes, bitmap); err != nil { + if err := bc.Engine().VerifyHeaderSignature(bc, block.Header(), sigBytes, bitmap); err != nil { return &sigVerifyErr{err} } - if err := chain.Engine().VerifyHeader(bc, block.Header(), true); err != nil { + if err := bc.Engine().VerifyHeader(bc, block.Header(), true); err != nil { return errors.Wrap(err, "[VerifyHeader]") } if _, err := bc.InsertChain(types.Blocks{block}, false); err != nil { diff --git a/internal/chain/engine.go b/internal/chain/engine.go index 4337c2932a..e01653646d 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -45,10 +45,10 @@ type engineImpl struct { verifiedSigCache *lru.Cache // verifiedSigKey -> struct{}{} } -var internal = NewEngine() +var internal engine.Engine = NewEngine() // NewEngine creates Engine with some cache -func NewEngine() *engineImpl { +func NewEngine() engine.Engine { sigCache, _ := lru.New(verifiedSigCache) epochCtxCache, _ := lru.New(epochCtxCache) return &engineImpl{ @@ -57,7 +57,7 @@ func NewEngine() *engineImpl { } } -func Engine() *engineImpl { +func Engine() engine.Engine { return internal } diff --git a/internal/shardchain/shardchains.go b/internal/shardchain/shardchains.go index 957ba9150b..771bfeb817 100644 --- a/internal/shardchain/shardchains.go +++ b/internal/shardchain/shardchains.go @@ -6,6 +6,7 @@ import ( "time" "github.com/harmony-one/harmony/core/state" + "github.com/harmony-one/harmony/internal/chain" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/shardchain/tikv_manage" @@ -133,7 +134,7 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c } if shardID == shard.BeaconChainShardID { bc, err = core.NewBlockChainWithOptions( - db, stateCache, bc, cacheConfig, &chainConfig, vm.Config{}, opts, + db, stateCache, bc, cacheConfig, &chainConfig, chain.Engine(), vm.Config{}, opts, ) } else { beacon, ok := sc.pool[shard.BeaconChainShardID] @@ -142,7 +143,7 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c } bc, err = core.NewBlockChainWithOptions( - db, stateCache, beacon, cacheConfig, &chainConfig, vm.Config{}, opts, + db, stateCache, beacon, cacheConfig, &chainConfig, chain.Engine(), vm.Config{}, opts, ) } } diff --git a/node/node_handler_test.go b/node/node_handler_test.go index b0d276d714..92ef236774 100644 --- a/node/node_handler_test.go +++ b/node/node_handler_test.go @@ -33,9 +33,8 @@ func TestAddNewBlock(t *testing.T) { if err != nil { t.Fatalf("newhost failure: %v", err) } - chainconfig := nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType().ChainConfig() collection := shardchain.NewCollection( - nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, &chainconfig, + nil, testDBFactory, &core.GenesisInitializer{}, nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType(), ) decider := quorum.NewDecider( quorum.SuperMajorityVote, shard.BeaconChainShardID, @@ -89,9 +88,8 @@ func TestVerifyNewBlock(t *testing.T) { if err != nil { t.Fatalf("newhost failure: %v", err) } - chainconfig := nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType().ChainConfig() collection := shardchain.NewCollection( - nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, &chainconfig, + nil, testDBFactory, &core.GenesisInitializer{}, nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType(), ) decider := quorum.NewDecider( quorum.SuperMajorityVote, shard.BeaconChainShardID, @@ -144,9 +142,8 @@ func TestVerifyVRF(t *testing.T) { if err != nil { t.Fatalf("newhost failure: %v", err) } - chainconfig := nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType().ChainConfig() collection := shardchain.NewCollection( - nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, &chainconfig, + nil, testDBFactory, &core.GenesisInitializer{}, nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType(), ) blockchain, err := collection.ShardChain(shard.BeaconChainShardID) if err != nil { diff --git a/node/node_newblock_test.go b/node/node_newblock_test.go index 5483e32c3b..4843d3f91d 100644 --- a/node/node_newblock_test.go +++ b/node/node_newblock_test.go @@ -34,9 +34,8 @@ func TestFinalizeNewBlockAsync(t *testing.T) { t.Fatalf("newhost failure: %v", err) } var testDBFactory = &shardchain.MemDBFactory{} - chainconfig := nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType().ChainConfig() collection := shardchain.NewCollection( - nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, &chainconfig, + nil, testDBFactory, &core.GenesisInitializer{}, nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType(), ) blockchain, err := collection.ShardChain(shard.BeaconChainShardID) require.NoError(t, err) diff --git a/node/node_test.go b/node/node_test.go index 86bd2b10bd..5f9d8ca51f 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -36,9 +36,8 @@ func TestNewNode(t *testing.T) { decider := quorum.NewDecider( quorum.SuperMajorityVote, shard.BeaconChainShardID, ) - chainconfig := nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType().ChainConfig() collection := shardchain.NewCollection( - nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, &chainconfig, + nil, testDBFactory, &core.GenesisInitializer{}, nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType(), ) blockchain, err := collection.ShardChain(shard.BeaconChainShardID) if err != nil { diff --git a/node/worker/worker_test.go b/node/worker/worker_test.go index 7bd25c8db8..3565be9542 100644 --- a/node/worker/worker_test.go +++ b/node/worker/worker_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/harmony-one/harmony/core/state" + "github.com/harmony-one/harmony/internal/chain" "github.com/ethereum/go-ethereum/core/rawdb" @@ -43,7 +44,7 @@ func TestNewWorker(t *testing.T) { genesis := gspec.MustCommit(database) _ = genesis - chain, err := core.NewBlockChain(database, state.NewDatabase(database), &core.BlockChainImpl{}, nil, gspec.Config, vm.Config{}) + chain, err := core.NewBlockChain(database, state.NewDatabase(database), &core.BlockChainImpl{}, nil, gspec.Config, chain.Engine(), vm.Config{}) if err != nil { t.Error(err) @@ -69,7 +70,7 @@ func TestCommitTransactions(t *testing.T) { ) gspec.MustCommit(database) - chain, _ := core.NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, vm.Config{}) + chain, _ := core.NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, chain.Engine(), vm.Config{}) // Create a new worker worker := New(params.TestChainConfig, chain, nil) diff --git a/test/chain/main.go b/test/chain/main.go index 6da9255692..6358e91806 100644 --- a/test/chain/main.go +++ b/test/chain/main.go @@ -206,7 +206,7 @@ func playFaucetContract(chain core.BlockChain) { func main() { genesis := gspec.MustCommit(database) - chain, _ := core.NewBlockChain(database, harmonyState.NewDatabase(database), nil, nil, gspec.Config, vm.Config{}) + chain, _ := core.NewBlockChain(database, harmonyState.NewDatabase(database), nil, nil, gspec.Config, chain.Engine(), vm.Config{}) txpool := core.NewTxPool(core.DefaultTxPoolConfig, chainConfig, chain, types.NewTransactionErrorSink()) backend := &testWorkerBackend{ diff --git a/test/chain/reward/main.go b/test/chain/reward/main.go index 828c6fc7f9..9afdeac39f 100644 --- a/test/chain/reward/main.go +++ b/test/chain/reward/main.go @@ -7,6 +7,7 @@ import ( "time" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/harmony-one/harmony/internal/chain" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/crypto/bls" @@ -107,7 +108,8 @@ func main() { database := rawdb.NewMemoryDatabase() genesis := gspec.MustCommit(database) _ = genesis - bc, _ := core.NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, vm.Config{}) + engine := chain.NewEngine() + bc, _ := core.NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, engine, vm.Config{}) statedb, _ := state.New(common2.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) msg := createValidator() statedb.AddBalance(msg.ValidatorAddress, new(big.Int).Mul(big.NewInt(5e18), big.NewInt(2000))) From fa84def721bae5fb71d91a547a0391ec851a9723 Mon Sep 17 00:00:00 2001 From: Casey Gardiner <117784577+ONECasey@users.noreply.github.com> Date: Tue, 28 Mar 2023 16:59:40 -0700 Subject: [PATCH 381/420] Revert "Removed engine dependency." (#4392) * Revert "Fix tests." This reverts commit 597ba2d6f1ed54ff599b9d9b8940c7285ab1277a. * Revert "Network type." This reverts commit 5e1878aedca3989dc0f34161dae1833e43ca6a76. * Revert "Clean up code." This reverts commit 15885f4c9b9263746827172b4f4f56d0926d18e2. * Revert "Fix possible panic." This reverts commit 1a70d5eb66cdbf8a23791806b71a323eed320085. * Revert "Removed engine dependency." This reverts commit 8c2ff803f709f944cfc8b1278f35cf5b2cacf859. --- api/service/legacysync/syncing.go | 4 ++-- api/service/stagedstreamsync/adapter.go | 1 + api/service/stagedstreamsync/sig_verify.go | 4 ++-- api/service/stagedsync/stage_state.go | 2 +- api/service/stagedsync/stagedsync.go | 4 ++-- block/header.go | 5 ----- cmd/harmony/main.go | 8 ++++++-- consensus/engine/consensus_engine.go | 2 +- consensus/engine/errors.go | 7 +++++++ consensus/validator.go | 2 +- core/block_validator.go | 16 +++++++-------- core/blockchain_impl.go | 6 +++--- core/epochchain.go | 7 +++++-- core/epochchain_test.go | 4 ++-- core/evm.go | 4 ++++ core/evm_test.go | 5 +++-- core/genesis_initializer.go | 9 ++++---- core/headerchain.go | 10 +++++++-- core/state_processor.go | 18 ++++++++-------- core/tx_pool_test.go | 6 ++++-- hmy/tracer.go | 7 +++---- internal/chain/engine.go | 10 ++------- internal/shardchain/dbinit.go | 7 ++----- internal/shardchain/shardchains.go | 24 ++++++++++++---------- node/node.go | 4 +++- node/node_cross_link.go | 3 +-- node/node_handler_test.go | 20 +++++++++++------- node/node_newblock_test.go | 7 +++++-- node/node_test.go | 7 +++++-- node/worker/worker.go | 24 ++++++++++++++++------ node/worker/worker_test.go | 12 ++++++----- test/chain/chain/chain_makers.go | 13 ++++++------ test/chain/main.go | 4 ++-- test/chain/reward/main.go | 2 +- 34 files changed, 157 insertions(+), 111 deletions(-) diff --git a/api/service/legacysync/syncing.go b/api/service/legacysync/syncing.go index 07ef9604e3..3375ccdc50 100644 --- a/api/service/legacysync/syncing.go +++ b/api/service/legacysync/syncing.go @@ -880,12 +880,12 @@ func (ss *StateSync) UpdateBlockAndStatus(block *types.Block, bc core.BlockChain } startTime := time.Now() - if err := chain.Engine().VerifyHeaderSignature(bc, block.Header(), sig, bitmap); err != nil { + if err := bc.Engine().VerifyHeaderSignature(bc, block.Header(), sig, bitmap); err != nil { return errors.Wrapf(err, "verify header signature %v", block.Hash().String()) } utils.Logger().Debug().Int64("elapsed time", time.Now().Sub(startTime).Milliseconds()).Msg("[Sync] VerifyHeaderSignature") } - err := chain.Engine().VerifyHeader(bc, block.Header(), verifySeal) + err := bc.Engine().VerifyHeader(bc, block.Header(), verifySeal) if err == engine.ErrUnknownAncestor { return err } else if err != nil { diff --git a/api/service/stagedstreamsync/adapter.go b/api/service/stagedstreamsync/adapter.go index bbd770c9b0..ae7632889c 100644 --- a/api/service/stagedstreamsync/adapter.go +++ b/api/service/stagedstreamsync/adapter.go @@ -27,6 +27,7 @@ type syncProtocol interface { type blockChain interface { engine.ChainReader + Engine() engine.Engine InsertChain(chain types.Blocks, verifyHeaders bool) (int, error) WriteCommitSig(blockNum uint64, lastCommits []byte) error diff --git a/api/service/stagedstreamsync/sig_verify.go b/api/service/stagedstreamsync/sig_verify.go index e50666bbde..649c6eaec1 100644 --- a/api/service/stagedstreamsync/sig_verify.go +++ b/api/service/stagedstreamsync/sig_verify.go @@ -47,10 +47,10 @@ func verifyAndInsertBlock(bc blockChain, block *types.Block, nextBlocks ...*type } } - if err := chain.Engine().VerifyHeaderSignature(bc, block.Header(), sigBytes, bitmap); err != nil { + if err := bc.Engine().VerifyHeaderSignature(bc, block.Header(), sigBytes, bitmap); err != nil { return &sigVerifyErr{err} } - if err := chain.Engine().VerifyHeader(bc, block.Header(), true); err != nil { + if err := bc.Engine().VerifyHeader(bc, block.Header(), true); err != nil { return errors.Wrap(err, "[VerifyHeader]") } if _, err := bc.InsertChain(types.Blocks{block}, false); err != nil { diff --git a/api/service/stagedsync/stage_state.go b/api/service/stagedsync/stage_state.go index 5492b890b5..7086acec18 100644 --- a/api/service/stagedsync/stage_state.go +++ b/api/service/stagedsync/stage_state.go @@ -250,7 +250,7 @@ func (stg *StageStates) verifyBlockSignatures(bc core.BlockChain, block *types.B } startTime := time.Now() - if err := chain.Engine().VerifyHeaderSignature(bc, block.Header(), sig, bitmap); err != nil { + if err := bc.Engine().VerifyHeaderSignature(bc, block.Header(), sig, bitmap); err != nil { return errors.Wrapf(err, "verify header signature %v", block.Hash().String()) } utils.Logger().Debug(). diff --git a/api/service/stagedsync/stagedsync.go b/api/service/stagedsync/stagedsync.go index 1080c97345..83af6abf9f 100644 --- a/api/service/stagedsync/stagedsync.go +++ b/api/service/stagedsync/stagedsync.go @@ -1052,14 +1052,14 @@ func (ss *StagedSync) UpdateBlockAndStatus(block *types.Block, bc core.BlockChai } startTime := time.Now() - if err := chain.Engine().VerifyHeaderSignature(bc, block.Header(), sig, bitmap); err != nil { + if err := bc.Engine().VerifyHeaderSignature(bc, block.Header(), sig, bitmap); err != nil { return errors.Wrapf(err, "verify header signature %v", block.Hash().String()) } utils.Logger().Debug(). Int64("elapsed time", time.Now().Sub(startTime).Milliseconds()). Msg("[STAGED_SYNC] VerifyHeaderSignature") } - err := chain.Engine().VerifyHeader(bc, block.Header(), verifySeal) + err := bc.Engine().VerifyHeader(bc, block.Header(), verifySeal) if err == engine.ErrUnknownAncestor { return err } else if err != nil { diff --git a/block/header.go b/block/header.go index 6991825430..5672446cc7 100644 --- a/block/header.go +++ b/block/header.go @@ -153,11 +153,6 @@ func (h *Header) IsLastBlockInEpoch() bool { return len(h.ShardState()) > 0 } -// NumberU64 returns the block number in uint64. -func (h Header) NumberU64() uint64 { - return h.Number().Uint64() -} - // HeaderRegistry is the taggedrlp type registry for versioned headers. var HeaderRegistry = taggedrlp.NewRegistry() diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index afffa1f3c6..f01cb758ed 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -16,6 +16,7 @@ import ( "time" "github.com/harmony-one/harmony/consensus/quorum" + "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/internal/registry" "github.com/harmony-one/harmony/internal/shardchain/tikv_manage" "github.com/harmony-one/harmony/internal/tikv/redis_helper" @@ -700,8 +701,11 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi chainDBFactory = &shardchain.LDBFactory{RootDir: nodeConfig.DBDir} } + engine := chain.NewEngine() + + chainConfig := nodeConfig.GetNetworkType().ChainConfig() collection := shardchain.NewCollection( - &hc, chainDBFactory, &core.GenesisInitializer{}, nodeConfig.GetNetworkType(), + &hc, chainDBFactory, &core.GenesisInitializer{NetworkType: nodeConfig.GetNetworkType()}, engine, &chainConfig, ) for shardID, archival := range nodeConfig.ArchiveModes() { if archival { @@ -736,7 +740,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi os.Exit(1) } - currentNode := node.New(myHost, currentConsensus, collection, blacklist, allowedTxs, localAccounts, nodeConfig.ArchiveModes(), &hc, registry) + currentNode := node.New(myHost, currentConsensus, engine, collection, blacklist, allowedTxs, localAccounts, nodeConfig.ArchiveModes(), &hc, registry) if hc.Legacy != nil && hc.Legacy.TPBroadcastInvalidTxn != nil { currentNode.BroadcastInvalidTx = *hc.Legacy.TPBroadcastInvalidTxn diff --git a/consensus/engine/consensus_engine.go b/consensus/engine/consensus_engine.go index 0879a5f546..3e7cc3b348 100644 --- a/consensus/engine/consensus_engine.go +++ b/consensus/engine/consensus_engine.go @@ -79,7 +79,7 @@ type Engine interface { // via the VerifySeal method. VerifyHeader(chain ChainReader, header *block.Header, seal bool) error - // VerifyHeaderSignature similar to VerifyHeader, which is only for verifying the block headers of one's own chain, this verification + // Similiar to VerifyHeader, which is only for verifying the block headers of one's own chain, this verification // is used for verifying "incoming" block header against commit signature and bitmap sent from the other chain cross-shard via libp2p. // i.e. this header verification api is more flexible since the caller specifies which commit signature and bitmap to use // for verifying the block header, which is necessary for cross-shard block header verification. Example of such is cross-shard transaction. diff --git a/consensus/engine/errors.go b/consensus/engine/errors.go index aab9614e9b..fe8e55f7bc 100644 --- a/consensus/engine/errors.go +++ b/consensus/engine/errors.go @@ -31,6 +31,13 @@ var ( // to the current node. ErrFutureBlock = errors.New("block in the future") + // ErrInvalidNumber is returned if a block's number doesn't equal it's parent's + // plus one. + ErrInvalidNumber = errors.New("invalid block number") + // ErrViewIDNotMatch is returned if the current viewID is not equal message's viewID ErrViewIDNotMatch = errors.New("viewID not match") + + // ErrInvalidConsensusMessage is returned is the consensus message received is invalid + ErrInvalidConsensusMessage = errors.New("invalid consensus message") ) diff --git a/consensus/validator.go b/consensus/validator.go index a270c665c6..f85cb8e3d8 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -331,7 +331,7 @@ func (consensus *Consensus) onCommitted(recvMsg *FBFTMessage) { Msg("[OnCommitted] Failed to parse commit sigBytes and bitmap") return } - if err := chain.Engine().VerifyHeaderSignature(consensus.Blockchain(), blockObj.Header(), + if err := consensus.Blockchain().Engine().VerifyHeaderSignature(consensus.Blockchain(), blockObj.Header(), sigBytes, bitmap); err != nil { consensus.getLogger().Error(). Uint64("blockNum", recvMsg.BlockNum). diff --git a/core/block_validator.go b/core/block_validator.go index 4a590c81e6..b1126f3972 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -26,8 +26,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/block" - "github.com/harmony-one/harmony/internal/chain" - chain2 "github.com/harmony-one/harmony/internal/chain" "github.com/pkg/errors" consensus_engine "github.com/harmony-one/harmony/consensus/engine" @@ -42,14 +40,16 @@ import ( // // BlockValidator implements validator. type BlockValidator struct { - config *params.ChainConfig // Chain configuration options - bc BlockChain // Canonical blockchain + config *params.ChainConfig // Chain configuration options + bc BlockChain // Canonical blockchain + engine consensus_engine.Engine // Consensus engine used for validating } // NewBlockValidator returns a new block validator which is safe for re-use -func NewBlockValidator(config *params.ChainConfig, blockchain BlockChain) *BlockValidator { +func NewBlockValidator(config *params.ChainConfig, blockchain BlockChain, engine consensus_engine.Engine) *BlockValidator { validator := &BlockValidator{ config: config, + engine: engine, bc: blockchain, } return validator @@ -131,7 +131,7 @@ func (v *BlockValidator) ValidateHeader(block *types.Block, seal bool) error { return errors.New("block is nil") } if h := block.Header(); h != nil { - return chain.Engine().VerifyHeader(v.bc, h, true) + return v.engine.VerifyHeader(v.bc, h, true) } return errors.New("header field was nil") } @@ -147,7 +147,7 @@ func (v *BlockValidator) ValidateHeaders(chain []*types.Block) (chan<- struct{}, headers[i] = block.Header() seals[i] = true } - return chain2.Engine().VerifyHeaders(v.bc, headers, seals) + return v.engine.VerifyHeaders(v.bc, headers, seals) } // CalcGasLimit computes the gas limit of the next block after parent. It aims @@ -249,5 +249,5 @@ func (v *BlockValidator) ValidateCXReceiptsProof(cxp *types.CXReceiptsProof) err // (4) verify blockHeader with seal var commitSig bls.SerializedSignature copy(commitSig[:], cxp.CommitSig) - return chain.Engine().VerifyHeaderSignature(v.bc, cxp.Header, commitSig, cxp.CommitBitmap) + return v.engine.VerifyHeaderSignature(v.bc, cxp.Header, commitSig, cxp.CommitBitmap) } diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 515c4099fa..8b0683bd65 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -275,7 +275,7 @@ func newBlockChainWithOptions( } var err error - bc.hc, err = NewHeaderChain(db, chainConfig, bc.getProcInterrupt) + bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.getProcInterrupt) if err != nil { return nil, err } @@ -294,8 +294,8 @@ func newBlockChainWithOptions( beaconChain = bc } - bc.SetValidator(NewBlockValidator(chainConfig, bc)) - bc.SetProcessor(NewStateProcessor(chainConfig, bc, beaconChain)) + bc.SetValidator(NewBlockValidator(chainConfig, bc, engine)) + bc.SetProcessor(NewStateProcessor(chainConfig, bc, beaconChain, engine)) // Take ownership of this particular state go bc.update() diff --git a/core/epochchain.go b/core/epochchain.go index ede5ec5a79..bcf00f5a89 100644 --- a/core/epochchain.go +++ b/core/epochchain.go @@ -9,6 +9,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/harmony-one/harmony/block" "github.com/harmony-one/harmony/consensus/engine" + consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" @@ -28,6 +29,7 @@ type EpochChain struct { db ethdb.Database // Low level persistent database to store final content in. mu chan struct{} currentHeader atomic.Value // Current head of the blockchain. + engine consensus_engine.Engine vmConfig *vm.Config headerCache *lru.Cache // Cache for the most recent block headers @@ -44,7 +46,7 @@ func cache(size int) *lru.Cache { } func NewEpochChain(db ethdb.Database, chainConfig *params.ChainConfig, - vmConfig vm.Config) (*EpochChain, error) { + engine consensus_engine.Engine, vmConfig vm.Config) (*EpochChain, error) { hash := rawdb.ReadCanonicalHash(db, 0) genesisBlock := rawdb.ReadBlock(db, hash, 0) @@ -58,6 +60,7 @@ func NewEpochChain(db ethdb.Database, chainConfig *params.ChainConfig, db: db, mu: make(chan struct{}, 1), currentHeader: atomic.Value{}, + engine: engine, vmConfig: &vmConfig, headerCache: cache(headerCacheLimit), @@ -240,7 +243,7 @@ func (bc *EpochChain) Config() *params.ChainConfig { } func (bc *EpochChain) Engine() engine.Engine { - return chain.Engine() + return bc.engine } func (bc *EpochChain) ReadShardState(epoch *big.Int) (*shard.State, error) { diff --git a/core/epochchain_test.go b/core/epochchain_test.go index 91c85bfdb3..eec7bd6280 100644 --- a/core/epochchain_test.go +++ b/core/epochchain_test.go @@ -12,10 +12,10 @@ import ( func TestGenesisBlock(t *testing.T) { db := rawdb.NewMemoryDatabase() - err := (&core.GenesisInitializer{}).InitChainDB(db, nodeconfig.Mainnet, 0) + err := (&core.GenesisInitializer{NetworkType: nodeconfig.Mainnet}).InitChainDB(db, 0) require.NoError(t, err) - chain, err := core.NewEpochChain(db, nil, vm.Config{}) + chain, err := core.NewEpochChain(db, nil, nil, vm.Config{}) require.NoError(t, err) header := chain.GetHeaderByNumber(0) diff --git a/core/evm.go b/core/evm.go index e11726a569..f395ab042f 100644 --- a/core/evm.go +++ b/core/evm.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/block" + consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/internal/params" @@ -38,6 +39,9 @@ import ( // ChainContext supports retrieving headers and consensus parameters from the // current blockchain to be used during transaction processing. type ChainContext interface { + // Engine retrieves the chain's consensus engine. + Engine() consensus_engine.Engine + // GetHeader returns the hash corresponding to their hash. GetHeader(common.Hash, uint64) *block.Header diff --git a/core/evm_test.go b/core/evm_test.go index 3201ee116b..962dedd307 100644 --- a/core/evm_test.go +++ b/core/evm_test.go @@ -21,7 +21,7 @@ import ( "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/crypto/hash" - "github.com/harmony-one/harmony/internal/chain" + chain2 "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/numeric" staking "github.com/harmony-one/harmony/staking/types" @@ -41,11 +41,12 @@ func getTestEnvironment(testBankKey ecdsa.PrivateKey) (*BlockChainImpl, *state.D Alloc: GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, ShardID: 0, } + engine = chain2.NewEngine() ) genesis := gspec.MustCommit(database) // fake blockchain - chain, _ := NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, chain.Engine(), vm.Config{}) + chain, _ := NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, engine, vm.Config{}) db, _ := chain.StateAt(genesis.Root()) // make a fake block header (use epoch 1 so that locked tokens can be tested) diff --git a/core/genesis_initializer.go b/core/genesis_initializer.go index 87ddd5fff3..04c6c85ed6 100644 --- a/core/genesis_initializer.go +++ b/core/genesis_initializer.go @@ -13,10 +13,11 @@ import ( // GenesisInitializer is a shardchain.DBInitializer adapter. type GenesisInitializer struct { + NetworkType nodeconfig.NetworkType } // InitChainDB sets up a new genesis block in the database for the given shard. -func (gi *GenesisInitializer) InitChainDB(db ethdb.Database, networkType nodeconfig.NetworkType, shardID uint32) error { +func (gi *GenesisInitializer) InitChainDB(db ethdb.Database, shardID uint32) error { shardState, _ := committee.WithStakingEnabled.Compute( big.NewInt(GenesisEpoch), nil, ) @@ -33,14 +34,14 @@ func (gi *GenesisInitializer) InitChainDB(db ethdb.Database, networkType nodecon } shardState = &shard.State{Shards: []shard.Committee{*subComm}} } - gi.setupGenesisBlock(db, shardID, shardState, networkType) + gi.setupGenesisBlock(db, shardID, shardState) return nil } // SetupGenesisBlock sets up a genesis blockchain. -func (gi *GenesisInitializer) setupGenesisBlock(db ethdb.Database, shardID uint32, myShardState *shard.State, networkType nodeconfig.NetworkType) { +func (gi *GenesisInitializer) setupGenesisBlock(db ethdb.Database, shardID uint32, myShardState *shard.State) { utils.Logger().Info().Interface("shardID", shardID).Msg("setting up a brand new chain database") - gspec := NewGenesisSpec(networkType, shardID) + gspec := NewGenesisSpec(gi.NetworkType, shardID) gspec.ShardStateHash = myShardState.Hash() gspec.ShardState = *myShardState.DeepCopy() // Store genesis block into db. diff --git a/core/headerchain.go b/core/headerchain.go index 72cdaa490b..a902d5a124 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -32,6 +32,7 @@ import ( lru "github.com/hashicorp/golang-lru" "github.com/harmony-one/harmony/block" + consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/utils" @@ -65,7 +66,8 @@ type HeaderChain struct { procInterrupt func() bool - rand *mrand.Rand + rand *mrand.Rand + engine consensus_engine.Engine } // NewHeaderChain creates a new HeaderChain structure. @@ -73,7 +75,7 @@ type HeaderChain struct { // getValidator should return the parent's validator // procInterrupt points to the parent's interrupt semaphore // wg points to the parent's shutdown wait group -func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, procInterrupt func() bool) (*HeaderChain, error) { +func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine consensus_engine.Engine, procInterrupt func() bool) (*HeaderChain, error) { headerCache, _ := lru.New(headerCacheLimit) tdCache, _ := lru.New(tdCacheLimit) numberCache, _ := lru.New(numberCacheLimit) @@ -94,6 +96,7 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, procInte canonicalCache: canonicalHash, procInterrupt: procInterrupt, rand: mrand.New(mrand.NewSource(seed.Int64())), + engine: engine, } hc.genesisHeader = hc.GetHeaderByNumber(0) @@ -545,6 +548,9 @@ func (hc *HeaderChain) SetGenesis(head *block.Header) { // Config retrieves the header chain's chain configuration. func (hc *HeaderChain) Config() *params.ChainConfig { return hc.config } +// Engine retrieves the header chain's consensus engine. +func (hc *HeaderChain) Engine() consensus_engine.Engine { return hc.engine } + // GetBlock implements consensus.ChainReader, and returns nil for every input as // a header chain does not have blocks available for retrieval. func (hc *HeaderChain) GetBlock(hash common.Hash, number uint64) *types.Block { diff --git a/core/state_processor.go b/core/state_processor.go index 5477130c8c..23c7e35389 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -20,13 +20,13 @@ import ( "math/big" "time" - "github.com/harmony-one/harmony/internal/chain" lru "github.com/hashicorp/golang-lru" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/block" + consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/consensus/reward" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" @@ -50,13 +50,14 @@ const ( // // StateProcessor implements Processor. type StateProcessor struct { - config *params.ChainConfig // Chain configuration options - bc BlockChain // Canonical blockchain - beacon BlockChain // Beacon chain - resultCache *lru.Cache // Cache for result after a certain block is processed + config *params.ChainConfig // Chain configuration options + bc BlockChain // Canonical blockchain + beacon BlockChain // Beacon chain + engine consensus_engine.Engine // Consensus engine used for block rewards + resultCache *lru.Cache // Cache for result after a certain block is processed } -// ProcessorResult structure is cached, and each individual element is returned +// this structure is cached, and each individual element is returned type ProcessorResult struct { Receipts types.Receipts CxReceipts types.CXReceipts @@ -69,7 +70,7 @@ type ProcessorResult struct { // NewStateProcessor initialises a new StateProcessor. func NewStateProcessor( - config *params.ChainConfig, bc BlockChain, beacon BlockChain, + config *params.ChainConfig, bc BlockChain, beacon BlockChain, engine consensus_engine.Engine, ) *StateProcessor { if bc == nil { panic("bc is nil") @@ -82,6 +83,7 @@ func NewStateProcessor( config: config, bc: bc, beacon: beacon, + engine: engine, resultCache: resultCache, } } @@ -193,7 +195,7 @@ func (p *StateProcessor) Process( // Block processing don't need to block on reward computation as in block proposal sigsReady <- true }() - _, payout, err := chain.Engine().Finalize( + _, payout, err := p.engine.Finalize( p.bc, p.beacon, header, statedb, block.Transactions(), diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index d6b181e182..fa3af1c972 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -28,8 +28,8 @@ import ( "time" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/harmony-one/harmony/crypto/bls" - "github.com/harmony-one/harmony/internal/chain" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" @@ -41,6 +41,7 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/crypto/hash" + chain2 "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/numeric" staking "github.com/harmony-one/harmony/staking/types" @@ -159,7 +160,8 @@ func createBlockChain() *BlockChainImpl { database := rawdb.NewMemoryDatabase() genesis := gspec.MustCommit(database) _ = genesis - blockchain, _ := NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, chain.Engine(), vm.Config{}) + engine := chain2.NewEngine() + blockchain, _ := NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, engine, vm.Config{}) return blockchain } diff --git a/hmy/tracer.go b/hmy/tracer.go index cdfa138aa6..699f784297 100644 --- a/hmy/tracer.go +++ b/hmy/tracer.go @@ -39,7 +39,6 @@ import ( "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/eth/rpc" "github.com/harmony-one/harmony/hmy/tracers" - "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/internal/utils" ) @@ -348,7 +347,7 @@ func (hmy *Harmony) TraceChain(ctx context.Context, start, end *types.Block, con // same as TraceBlock, but only use 1 thread func (hmy *Harmony) traceBlockNoThread(ctx context.Context, block *types.Block, config *TraceConfig) ([]*TxTraceResult, error) { // Create the parent state database - if err := chain.Engine().VerifyHeader(hmy.BlockChain, block.Header(), true); err != nil { + if err := hmy.BlockChain.Engine().VerifyHeader(hmy.BlockChain, block.Header(), true); err != nil { return nil, err } parent := hmy.BlockChain.GetBlock(block.ParentHash(), block.NumberU64()-1) @@ -423,7 +422,7 @@ func (hmy *Harmony) TraceBlock(ctx context.Context, block *types.Block, config * return hmy.traceBlockNoThread(ctx, block, config) } // Create the parent state database - if err := chain.Engine().VerifyHeader(hmy.BlockChain, block.Header(), true); err != nil { + if err := hmy.BlockChain.Engine().VerifyHeader(hmy.BlockChain, block.Header(), true); err != nil { return nil, err } parent := hmy.BlockChain.GetBlock(block.ParentHash(), block.NumberU64()-1) @@ -524,7 +523,7 @@ func (hmy *Harmony) standardTraceBlockToFile(ctx context.Context, block *types.B } } // Create the parent state database - if err := chain.Engine().VerifyHeader(hmy.BlockChain, block.Header(), true); err != nil { + if err := hmy.BlockChain.Engine().VerifyHeader(hmy.BlockChain, block.Header(), true); err != nil { return nil, err } parent := hmy.BlockChain.GetBlock(block.ParentHash(), block.NumberU64()-1) diff --git a/internal/chain/engine.go b/internal/chain/engine.go index e01653646d..6e018dccfd 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -6,7 +6,6 @@ import ( "sort" "time" - "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/internal/params" bls2 "github.com/harmony-one/bls/ffi/go/bls" @@ -17,6 +16,7 @@ import ( "github.com/pkg/errors" "github.com/harmony-one/harmony/block" + "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/consensus/reward" "github.com/harmony-one/harmony/consensus/signature" @@ -45,10 +45,8 @@ type engineImpl struct { verifiedSigCache *lru.Cache // verifiedSigKey -> struct{}{} } -var internal engine.Engine = NewEngine() - // NewEngine creates Engine with some cache -func NewEngine() engine.Engine { +func NewEngine() *engineImpl { sigCache, _ := lru.New(verifiedSigCache) epochCtxCache, _ := lru.New(epochCtxCache) return &engineImpl{ @@ -57,10 +55,6 @@ func NewEngine() engine.Engine { } } -func Engine() engine.Engine { - return internal -} - // VerifyHeader checks whether a header conforms to the consensus rules of the bft engine. // Note that each block header contains the bls signature of the parent block func (e *engineImpl) VerifyHeader(chain engine.ChainReader, header *block.Header, seal bool) error { diff --git a/internal/shardchain/dbinit.go b/internal/shardchain/dbinit.go index b7b5d9e38a..4987a78862 100644 --- a/internal/shardchain/dbinit.go +++ b/internal/shardchain/dbinit.go @@ -1,11 +1,8 @@ package shardchain -import ( - "github.com/ethereum/go-ethereum/ethdb" - nodeconfig "github.com/harmony-one/harmony/internal/configs/node" -) +import "github.com/ethereum/go-ethereum/ethdb" // DBInitializer initializes a newly created chain database. type DBInitializer interface { - InitChainDB(db ethdb.Database, networkType nodeconfig.NetworkType, shardID uint32) error + InitChainDB(db ethdb.Database, shardID uint32) error } diff --git a/internal/shardchain/shardchains.go b/internal/shardchain/shardchains.go index 771bfeb817..1be2c68414 100644 --- a/internal/shardchain/shardchains.go +++ b/internal/shardchain/shardchains.go @@ -6,18 +6,18 @@ import ( "time" "github.com/harmony-one/harmony/core/state" - "github.com/harmony-one/harmony/internal/chain" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" - nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/shardchain/tikv_manage" "github.com/harmony-one/harmony/shard" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" + "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/vm" + "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/utils" "github.com/pkg/errors" ) @@ -40,10 +40,11 @@ type Collection interface { type CollectionImpl struct { dbFactory DBFactory dbInit DBInitializer + engine engine.Engine mtx sync.Mutex pool map[uint32]core.BlockChain disableCache map[uint32]bool - networkType nodeconfig.NetworkType + chainConfig *params.ChainConfig harmonyconfig *harmonyconfig.HarmonyConfig } @@ -55,16 +56,17 @@ type CollectionImpl struct { // the factory is brand new (empty). func NewCollection( harmonyconfig *harmonyconfig.HarmonyConfig, - dbFactory DBFactory, dbInit DBInitializer, - network nodeconfig.NetworkType, + dbFactory DBFactory, dbInit DBInitializer, engine engine.Engine, + chainConfig *params.ChainConfig, ) *CollectionImpl { return &CollectionImpl{ harmonyconfig: harmonyconfig, dbFactory: dbFactory, dbInit: dbInit, + engine: engine, pool: make(map[uint32]core.BlockChain), disableCache: make(map[uint32]bool), - networkType: network, + chainConfig: chainConfig, } } @@ -93,7 +95,7 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c utils.Logger().Info(). Uint32("shardID", shardID). Msg("initializing a new chain database") - if err := sc.dbInit.InitChainDB(db, sc.networkType, shardID); err != nil { + if err := sc.dbInit.InitChainDB(db, shardID); err != nil { return nil, errors.Wrapf(err, "cannot initialize a new chain database") } } @@ -114,7 +116,7 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c } } - chainConfig := sc.networkType.ChainConfig() + chainConfig := *sc.chainConfig if shardID == shard.BeaconChainShardID { // For beacon chain inside a shard chain, need to reset the eth chainID to shard 0's eth chainID in the config @@ -126,7 +128,7 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c } var bc core.BlockChain if opts.EpochChain { - bc, err = core.NewEpochChain(db, &chainConfig, vm.Config{}) + bc, err = core.NewEpochChain(db, &chainConfig, sc.engine, vm.Config{}) } else { stateCache, err := initStateCache(db, sc, shardID) if err != nil { @@ -134,7 +136,7 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c } if shardID == shard.BeaconChainShardID { bc, err = core.NewBlockChainWithOptions( - db, stateCache, bc, cacheConfig, &chainConfig, chain.Engine(), vm.Config{}, opts, + db, stateCache, bc, cacheConfig, &chainConfig, sc.engine, vm.Config{}, opts, ) } else { beacon, ok := sc.pool[shard.BeaconChainShardID] @@ -143,7 +145,7 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c } bc, err = core.NewBlockChainWithOptions( - db, stateCache, beacon, cacheConfig, &chainConfig, chain.Engine(), vm.Config{}, opts, + db, stateCache, beacon, cacheConfig, &chainConfig, sc.engine, vm.Config{}, opts, ) } } diff --git a/node/node.go b/node/node.go index 360a885e99..13988e2ad3 100644 --- a/node/node.go +++ b/node/node.go @@ -12,6 +12,7 @@ import ( "sync" "time" + "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/internal/registry" "github.com/harmony-one/harmony/internal/shardchain/tikv_manage" "github.com/harmony-one/harmony/internal/tikv" @@ -1014,6 +1015,7 @@ func (node *Node) GetSyncID() [SyncIDLength]byte { func New( host p2p.Host, consensusObj *consensus.Consensus, + engine engine.Engine, collection *shardchain.CollectionImpl, blacklist map[common.Address]struct{}, allowedTxs map[common.Address]core.AllowedTxData, @@ -1108,7 +1110,7 @@ func New( node.TxPool = core.NewTxPool(txPoolConfig, node.Blockchain().Config(), blockchain, node.TransactionErrorSink) node.CxPool = core.NewCxPool(core.CxPoolSize) - node.Worker = worker.New(node.Blockchain().Config(), blockchain, beaconChain) + node.Worker = worker.New(node.Blockchain().Config(), blockchain, beaconChain, engine) node.deciderCache, _ = lru.New(16) node.committeeCache, _ = lru.New(16) diff --git a/node/node_cross_link.go b/node/node_cross_link.go index 0a919368d1..d0f80545d5 100644 --- a/node/node_cross_link.go +++ b/node/node_cross_link.go @@ -8,7 +8,6 @@ import ( ffi_bls "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" - "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" "github.com/pkg/errors" @@ -180,7 +179,7 @@ func (node *Node) VerifyCrossLink(cl types.CrossLink) error { if cl.ShardID() >= instance.NumShards() { return errors.New("[VerifyCrossLink] ShardID should less than NumShards") } - engine := chain.Engine() + engine := node.Blockchain().Engine() if err := engine.VerifyCrossLink(node.Blockchain(), cl); err != nil { return errors.Wrap(err, "[VerifyCrossLink]") diff --git a/node/node_handler_test.go b/node/node_handler_test.go index 92ef236774..25d966f469 100644 --- a/node/node_handler_test.go +++ b/node/node_handler_test.go @@ -33,8 +33,10 @@ func TestAddNewBlock(t *testing.T) { if err != nil { t.Fatalf("newhost failure: %v", err) } + engine := chain.NewEngine() + chainconfig := nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType().ChainConfig() collection := shardchain.NewCollection( - nil, testDBFactory, &core.GenesisInitializer{}, nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType(), + nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, engine, &chainconfig, ) decider := quorum.NewDecider( quorum.SuperMajorityVote, shard.BeaconChainShardID, @@ -51,7 +53,7 @@ func TestAddNewBlock(t *testing.T) { t.Fatalf("Cannot craeate consensus: %v", err) } nodeconfig.SetNetworkType(nodeconfig.Devnet) - node := New(host, consensus, collection, nil, nil, nil, nil, nil, reg) + node := New(host, consensus, engine, collection, nil, nil, nil, nil, nil, reg) txs := make(map[common.Address]types.Transactions) stks := staking.StakingTransactions{} @@ -88,8 +90,10 @@ func TestVerifyNewBlock(t *testing.T) { if err != nil { t.Fatalf("newhost failure: %v", err) } + engine := chain.NewEngine() + chainconfig := nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType().ChainConfig() collection := shardchain.NewCollection( - nil, testDBFactory, &core.GenesisInitializer{}, nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType(), + nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, engine, &chainconfig, ) decider := quorum.NewDecider( quorum.SuperMajorityVote, shard.BeaconChainShardID, @@ -108,7 +112,7 @@ func TestVerifyNewBlock(t *testing.T) { archiveMode := make(map[uint32]bool) archiveMode[0] = true archiveMode[1] = false - node := New(host, consensus, collection, nil, nil, nil, archiveMode, nil, reg) + node := New(host, consensus, engine, collection, nil, nil, nil, archiveMode, nil, reg) txs := make(map[common.Address]types.Transactions) stks := staking.StakingTransactions{} @@ -142,8 +146,10 @@ func TestVerifyVRF(t *testing.T) { if err != nil { t.Fatalf("newhost failure: %v", err) } + engine := chain.NewEngine() + chainconfig := nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType().ChainConfig() collection := shardchain.NewCollection( - nil, testDBFactory, &core.GenesisInitializer{}, nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType(), + nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, engine, &chainconfig, ) blockchain, err := collection.ShardChain(shard.BeaconChainShardID) if err != nil { @@ -162,7 +168,7 @@ func TestVerifyVRF(t *testing.T) { archiveMode := make(map[uint32]bool) archiveMode[0] = true archiveMode[1] = false - node := New(host, consensus, collection, nil, nil, nil, archiveMode, nil, reg) + node := New(host, consensus, engine, collection, nil, nil, nil, archiveMode, nil, reg) txs := make(map[common.Address]types.Transactions) stks := staking.StakingTransactions{} @@ -199,7 +205,7 @@ func TestVerifyVRF(t *testing.T) { node.Blockchain().WriteShardStateBytes(node.Blockchain().ChainDb(), big.NewInt(1), node.Worker.GetCurrentHeader().ShardState()) node.Blockchain().Config().VRFEpoch = big.NewInt(0) - if err := chain.Engine().VerifyVRF( + if err := node.Blockchain().Engine().VerifyVRF( node.Blockchain(), block.Header(), ); err != nil { t.Error("New vrf is not verified successfully:", err) diff --git a/node/node_newblock_test.go b/node/node_newblock_test.go index 4843d3f91d..492175b1d2 100644 --- a/node/node_newblock_test.go +++ b/node/node_newblock_test.go @@ -10,6 +10,7 @@ import ( "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/crypto/bls" + "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/registry" "github.com/harmony-one/harmony/internal/shardchain" @@ -34,8 +35,10 @@ func TestFinalizeNewBlockAsync(t *testing.T) { t.Fatalf("newhost failure: %v", err) } var testDBFactory = &shardchain.MemDBFactory{} + engine := chain.NewEngine() + chainconfig := nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType().ChainConfig() collection := shardchain.NewCollection( - nil, testDBFactory, &core.GenesisInitializer{}, nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType(), + nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, engine, &chainconfig, ) blockchain, err := collection.ShardChain(shard.BeaconChainShardID) require.NoError(t, err) @@ -50,7 +53,7 @@ func TestFinalizeNewBlockAsync(t *testing.T) { t.Fatalf("Cannot craeate consensus: %v", err) } - node := New(host, consensus, collection, nil, nil, nil, nil, nil, registry.New().SetBlockchain(blockchain)) + node := New(host, consensus, engine, collection, nil, nil, nil, nil, nil, registry.New().SetBlockchain(blockchain)) node.Worker.UpdateCurrent() diff --git a/node/node_test.go b/node/node_test.go index 5f9d8ca51f..49ba5d164d 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -8,6 +8,7 @@ import ( "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/crypto/bls" + "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/registry" "github.com/harmony-one/harmony/internal/shardchain" @@ -33,11 +34,13 @@ func TestNewNode(t *testing.T) { if err != nil { t.Fatalf("newhost failure: %v", err) } + engine := chain.NewEngine() decider := quorum.NewDecider( quorum.SuperMajorityVote, shard.BeaconChainShardID, ) + chainconfig := nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType().ChainConfig() collection := shardchain.NewCollection( - nil, testDBFactory, &core.GenesisInitializer{}, nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType(), + nil, testDBFactory, &core.GenesisInitializer{NetworkType: nodeconfig.GetShardConfig(shard.BeaconChainShardID).GetNetworkType()}, engine, &chainconfig, ) blockchain, err := collection.ShardChain(shard.BeaconChainShardID) if err != nil { @@ -51,7 +54,7 @@ func TestNewNode(t *testing.T) { t.Fatalf("Cannot craeate consensus: %v", err) } - node := New(host, consensus, collection, nil, nil, nil, nil, nil, reg) + node := New(host, consensus, engine, collection, nil, nil, nil, nil, nil, reg) if node.Consensus == nil { t.Error("Consensus is not initialized for the node") } diff --git a/node/worker/worker.go b/node/worker/worker.go index d2a8f407a4..1a711bc193 100644 --- a/node/worker/worker.go +++ b/node/worker/worker.go @@ -8,7 +8,6 @@ import ( "time" "github.com/harmony-one/harmony/consensus/reward" - "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/consensus" @@ -20,6 +19,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/block" blockfactory "github.com/harmony-one/harmony/block/factory" + consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" @@ -59,6 +59,7 @@ type Worker struct { chain core.BlockChain beacon core.BlockChain current *environment // An environment for current running cycle. + engine consensus_engine.Engine gasFloor uint64 gasCeil uint64 } @@ -311,7 +312,7 @@ func (w *Worker) UpdateCurrent() error { Time(big.NewInt(timestamp)). ShardID(w.chain.ShardID()). Header() - return w.makeCurrent(parent.Header(), header) + return w.makeCurrent(parent, header) } // GetCurrentHeader returns the current header to propose @@ -320,7 +321,7 @@ func (w *Worker) GetCurrentHeader() *block.Header { } // makeCurrent creates a new environment for the current cycle. -func (w *Worker) makeCurrent(parent *block.Header, header *block.Header) error { +func (w *Worker) makeCurrent(parent *types.Block, header *block.Header) error { state, err := w.chain.StateAt(parent.Root()) if err != nil { return err @@ -380,6 +381,16 @@ func (w *Worker) GetCurrentReceipts() []*types.Receipt { return w.current.receipts } +// OutgoingReceipts get the receipts generated starting from the last state. +func (w *Worker) OutgoingReceipts() []*types.CXReceipt { + return w.current.outcxs +} + +// IncomingReceipts get incoming receipts in destination shard that is received from source shard +func (w *Worker) IncomingReceipts() []*types.CXReceiptsProof { + return w.current.incxs +} + // CollectVerifiedSlashes sets w.current.slashes only to those that // past verification func (w *Worker) CollectVerifiedSlashes() error { @@ -547,7 +558,7 @@ func (w *Worker) FinalizeNewBlock( } }() - block, payout, err := chain.Engine().Finalize( + block, payout, err := w.engine.Finalize( w.chain, w.beacon, copyHeader, state, w.current.txs, w.current.receipts, @@ -563,13 +574,14 @@ func (w *Worker) FinalizeNewBlock( // New create a new worker object. func New( - config *params.ChainConfig, chain core.BlockChain, beacon core.BlockChain, + config *params.ChainConfig, chain core.BlockChain, beacon core.BlockChain, engine consensus_engine.Engine, ) *Worker { worker := &Worker{ config: config, factory: blockfactory.NewFactory(config), chain: chain, beacon: beacon, + engine: engine, } worker.gasFloor = 80000000 worker.gasCeil = 120000000 @@ -586,7 +598,7 @@ func New( Time(big.NewInt(timestamp)). ShardID(worker.chain.ShardID()). Header() - worker.makeCurrent(parent.Header(), header) + worker.makeCurrent(parent, header) return worker } diff --git a/node/worker/worker_test.go b/node/worker/worker_test.go index 3565be9542..e6ffdfddad 100644 --- a/node/worker/worker_test.go +++ b/node/worker/worker_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/harmony-one/harmony/core/state" - "github.com/harmony-one/harmony/internal/chain" "github.com/ethereum/go-ethereum/core/rawdb" @@ -17,6 +16,7 @@ import ( "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" + chain2 "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/internal/params" ) @@ -40,17 +40,18 @@ func TestNewWorker(t *testing.T) { Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, ShardID: 10, } + engine = chain2.NewEngine() ) genesis := gspec.MustCommit(database) _ = genesis - chain, err := core.NewBlockChain(database, state.NewDatabase(database), &core.BlockChainImpl{}, nil, gspec.Config, chain.Engine(), vm.Config{}) + chain, err := core.NewBlockChain(database, state.NewDatabase(database), &core.BlockChainImpl{}, nil, gspec.Config, engine, vm.Config{}) if err != nil { t.Error(err) } // Create a new worker - worker := New(params.TestChainConfig, chain, nil) + worker := New(params.TestChainConfig, chain, nil, engine) if worker.GetCurrentState().GetBalance(crypto.PubkeyToAddress(testBankKey.PublicKey)).Cmp(testBankFunds) != 0 { t.Error("Worker state is not setup correctly") @@ -67,13 +68,14 @@ func TestCommitTransactions(t *testing.T) { Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, ShardID: 0, } + engine = chain2.NewEngine() ) gspec.MustCommit(database) - chain, _ := core.NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, chain.Engine(), vm.Config{}) + chain, _ := core.NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, engine, vm.Config{}) // Create a new worker - worker := New(params.TestChainConfig, chain, nil) + worker := New(params.TestChainConfig, chain, nil, engine) // Generate a test tx baseNonce := worker.GetCurrentState().GetNonce(crypto.PubkeyToAddress(testBankKey.PublicKey)) diff --git a/test/chain/chain/chain_makers.go b/test/chain/chain/chain_makers.go index d5c10a5d59..b4ca9ecd7c 100644 --- a/test/chain/chain/chain_makers.go +++ b/test/chain/chain/chain_makers.go @@ -23,7 +23,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/harmony-one/harmony/core" - "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/block" blockfactory "github.com/harmony-one/harmony/block/factory" @@ -50,6 +49,7 @@ type BlockGen struct { receipts []*types.Receipt uncles []*block.Header config *params.ChainConfig + engine consensus_engine.Engine } // SetCoinbase sets the coinbase of the generated block. @@ -167,7 +167,7 @@ func (b *BlockGen) PrevBlock(index int) *types.Block { // a similar non-validating proof of work implementation. func GenerateChain( config *params.ChainConfig, parent *types.Block, - db ethdb.Database, + engine consensus_engine.Engine, db ethdb.Database, n int, gen func(int, *BlockGen), ) ([]*types.Block, []types.Receipts) { @@ -185,17 +185,18 @@ func GenerateChain( statedb: statedb, config: config, factory: factory, + engine: engine, } - b.header = makeHeader(chainreader, parent, statedb, factory) + b.header = makeHeader(chainreader, parent, statedb, b.engine, factory) // Execute any user modifications to the block if gen != nil { gen(i, b) } - if true { + if b.engine != nil { // Finalize and seal the block - block, _, err := chain.Engine().Finalize( + block, _, err := b.engine.Finalize( chainreader, nil, b.header, statedb, b.txs, b.receipts, nil, nil, nil, nil, nil, func() uint64 { return 0 }, ) if err != nil { @@ -227,7 +228,7 @@ func GenerateChain( return blocks, receipts } -func makeHeader(chain consensus_engine.ChainReader, parent *types.Block, state *state.DB, factory blockfactory.Factory) *block.Header { +func makeHeader(chain consensus_engine.ChainReader, parent *types.Block, state *state.DB, engine consensus_engine.Engine, factory blockfactory.Factory) *block.Header { var time *big.Int if parent.Time() == nil { time = big.NewInt(10) diff --git a/test/chain/main.go b/test/chain/main.go index 6358e91806..e78b519356 100644 --- a/test/chain/main.go +++ b/test/chain/main.go @@ -94,7 +94,7 @@ func fundFaucetContract(chain core.BlockChain) { fmt.Println("--------- Funding addresses for Faucet Contract Call ---------") fmt.Println() - contractworker = pkgworker.New(params.TestChainConfig, chain, nil) + contractworker = pkgworker.New(params.TestChainConfig, chain, nil, chain.Engine()) nonce = contractworker.GetCurrentState().GetNonce(crypto.PubkeyToAddress(FaucetPriKey.PublicKey)) dataEnc = common.FromHex(FaucetContractBinary) ftx, _ := types.SignTx( @@ -223,7 +223,7 @@ func main() { //// Generate a small n-block chain and an uncle block for it n := 3 if n > 0 { - blocks, _ := chain2.GenerateChain(chainConfig, genesis, database, n, func(i int, gen *chain2.BlockGen) { + blocks, _ := chain2.GenerateChain(chainConfig, genesis, chain.Engine(), database, n, func(i int, gen *chain2.BlockGen) { gen.SetCoinbase(FaucetAddress) gen.SetShardID(0) gen.AddTx(pendingTxs[i].(*types.Transaction)) diff --git a/test/chain/reward/main.go b/test/chain/reward/main.go index 9afdeac39f..3848697b66 100644 --- a/test/chain/reward/main.go +++ b/test/chain/reward/main.go @@ -7,7 +7,6 @@ import ( "time" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/harmony-one/harmony/internal/chain" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/crypto/bls" @@ -23,6 +22,7 @@ import ( "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/crypto/hash" + "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/internal/common" protobuf "github.com/golang/protobuf/proto" From 3226f31211d1643659ca7b3f0ff36b6c18125706 Mon Sep 17 00:00:00 2001 From: Max <82761650+MaxMustermann2@users.noreply.github.com> Date: Wed, 29 Mar 2023 00:00:36 +0000 Subject: [PATCH 382/420] gitignore the cache folder (#4389) --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6a3b3c4267..bbdb537722 100644 --- a/.gitignore +++ b/.gitignore @@ -96,4 +96,5 @@ explorer_storage_* profiles/*.pb.gz # cache db -cache_*_db \ No newline at end of file +cache/ +cache_*_db From 49609d99d590e827e379cbc0492c9c5b196e185e Mon Sep 17 00:00:00 2001 From: Soph <35721420+sophoah@users.noreply.github.com> Date: Wed, 29 Mar 2023 07:00:51 +0700 Subject: [PATCH 383/420] stable localnet with external validator (#4388) * stable localnet with external validator * ignore deploy config file comments * reduce node launched in localnet * update makefile * localnet configuration - add more fn * fix validator information command typo --- ...6a4f5e048e6d8b8a000d77a478d44cd640270c.key | 1 + ...a4f5e048e6d8b8a000d77a478d44cd640270c.pass | 0 ...ad1e86dce7458b1c295404fb54a0d61e50bb97.key | 1 + ...d1e86dce7458b1c295404fb54a0d61e50bb97.pass | 0 ...19d3f37e70ba1cd8d082cf8ff7be2f861db48c.key | 1 + ...9d3f37e70ba1cd8d082cf8ff7be2f861db48c.pass | 0 ...0535d646120171961d74968d27a2ec0f8af285.key | 1 + ...535d646120171961d74968d27a2ec0f8af285.pass | 0 ...0654ef8a727710ee55ad8b532da0dd87811915.key | 1 + ...654ef8a727710ee55ad8b532da0dd87811915.pass | 0 ...3d9a3b16c4a55f84522f553872225a7b1efc0c.key | 1 + ...d9a3b16c4a55f84522f553872225a7b1efc0c.pass | 0 ...7ughrllgnzx9sfa46p568k8rdmtz7qj85slc6t.key | 1 + ...9aw2wcr5y4lxeuwt0ajgt5aw3a3qkjdgg67ygj.key | 1 + ...9zzwsxr0uf2fe34y8qkadek2v0eh6h5mg2deg6.key | 1 + ...auqndgthqu5lznsn7tuma8s5333cq0y07cwc6x.key | 1 + ...eenp9ujcrmyaq22ef6jrpry2k97tjz4xs6ppcf.key | 1 + ...lctumupg2y009pjmnhnmn4nqjk0zf0dspjanf7.key | 1 + Makefile | 8 ++- internal/configs/sharding/localnet.go | 4 +- test/build-localnet-validator.sh | 70 +++++++++++++++++++ .../local-resharding-with-external.txt | 37 ++++++++++ test/debug-external.sh | 8 +++ test/deploy.sh | 4 +- 24 files changed, 137 insertions(+), 6 deletions(-) create mode 100644 .hmy/extbls/4f41a37a3a8d0695dd6edcc58142c6b7d98e74da5c90e79b587b3b960b6a4f5e048e6d8b8a000d77a478d44cd640270c.key create mode 100644 .hmy/extbls/4f41a37a3a8d0695dd6edcc58142c6b7d98e74da5c90e79b587b3b960b6a4f5e048e6d8b8a000d77a478d44cd640270c.pass create mode 100644 .hmy/extbls/5a18d4aa3e6aff4835f07588ae66be19684476d38799f63e54c6b5732fad1e86dce7458b1c295404fb54a0d61e50bb97.key create mode 100644 .hmy/extbls/5a18d4aa3e6aff4835f07588ae66be19684476d38799f63e54c6b5732fad1e86dce7458b1c295404fb54a0d61e50bb97.pass create mode 100644 .hmy/extbls/7dcc035a943e29e17959dabe636efad7303d2c6f273ace457ba9dcc2fd19d3f37e70ba1cd8d082cf8ff7be2f861db48c.key create mode 100644 .hmy/extbls/7dcc035a943e29e17959dabe636efad7303d2c6f273ace457ba9dcc2fd19d3f37e70ba1cd8d082cf8ff7be2f861db48c.pass create mode 100644 .hmy/extbls/81296eedba05047594385e3086e1dab52c9eb9e56f46d86f58447cccc20535d646120171961d74968d27a2ec0f8af285.key create mode 100644 .hmy/extbls/81296eedba05047594385e3086e1dab52c9eb9e56f46d86f58447cccc20535d646120171961d74968d27a2ec0f8af285.pass create mode 100644 .hmy/extbls/89eab762e7364d6cf89f7a6c54da794f74eba2e29147992ac66adcef0f0654ef8a727710ee55ad8b532da0dd87811915.key create mode 100644 .hmy/extbls/89eab762e7364d6cf89f7a6c54da794f74eba2e29147992ac66adcef0f0654ef8a727710ee55ad8b532da0dd87811915.pass create mode 100644 .hmy/extbls/b0917378b179a519a5055259c4f8980cce37d58af300b00dd98b07076d3d9a3b16c4a55f84522f553872225a7b1efc0c.key create mode 100644 .hmy/extbls/b0917378b179a519a5055259c4f8980cce37d58af300b00dd98b07076d3d9a3b16c4a55f84522f553872225a7b1efc0c.pass create mode 100644 .hmy/extkeystore/one17ughrllgnzx9sfa46p568k8rdmtz7qj85slc6t.key create mode 100644 .hmy/extkeystore/one19aw2wcr5y4lxeuwt0ajgt5aw3a3qkjdgg67ygj.key create mode 100644 .hmy/extkeystore/one19zzwsxr0uf2fe34y8qkadek2v0eh6h5mg2deg6.key create mode 100644 .hmy/extkeystore/one1auqndgthqu5lznsn7tuma8s5333cq0y07cwc6x.key create mode 100644 .hmy/extkeystore/one1eenp9ujcrmyaq22ef6jrpry2k97tjz4xs6ppcf.key create mode 100644 .hmy/extkeystore/one1lctumupg2y009pjmnhnmn4nqjk0zf0dspjanf7.key create mode 100644 test/build-localnet-validator.sh create mode 100644 test/configs/local-resharding-with-external.txt create mode 100755 test/debug-external.sh diff --git a/.hmy/extbls/4f41a37a3a8d0695dd6edcc58142c6b7d98e74da5c90e79b587b3b960b6a4f5e048e6d8b8a000d77a478d44cd640270c.key b/.hmy/extbls/4f41a37a3a8d0695dd6edcc58142c6b7d98e74da5c90e79b587b3b960b6a4f5e048e6d8b8a000d77a478d44cd640270c.key new file mode 100644 index 0000000000..77581cba0b --- /dev/null +++ b/.hmy/extbls/4f41a37a3a8d0695dd6edcc58142c6b7d98e74da5c90e79b587b3b960b6a4f5e048e6d8b8a000d77a478d44cd640270c.key @@ -0,0 +1 @@ +a010bc1550956a3a4935cdc6f8633ea770bd0321094cfaccd25d3ea3b5382417cb5c150de0357ee8b6eeeec135ae0ec379701c7f35ae33da9030e93a630dc3a988577d1a1b0a9e60976f79d5cefa68123ac77c636f2ccab149fc8fc7 \ No newline at end of file diff --git a/.hmy/extbls/4f41a37a3a8d0695dd6edcc58142c6b7d98e74da5c90e79b587b3b960b6a4f5e048e6d8b8a000d77a478d44cd640270c.pass b/.hmy/extbls/4f41a37a3a8d0695dd6edcc58142c6b7d98e74da5c90e79b587b3b960b6a4f5e048e6d8b8a000d77a478d44cd640270c.pass new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.hmy/extbls/5a18d4aa3e6aff4835f07588ae66be19684476d38799f63e54c6b5732fad1e86dce7458b1c295404fb54a0d61e50bb97.key b/.hmy/extbls/5a18d4aa3e6aff4835f07588ae66be19684476d38799f63e54c6b5732fad1e86dce7458b1c295404fb54a0d61e50bb97.key new file mode 100644 index 0000000000..ae7af7f0ee --- /dev/null +++ b/.hmy/extbls/5a18d4aa3e6aff4835f07588ae66be19684476d38799f63e54c6b5732fad1e86dce7458b1c295404fb54a0d61e50bb97.key @@ -0,0 +1 @@ +e176a1a8d9c533ad7efbaa9caca1d839146495683f669be35dddfebe2f39497c73fc2a6d7da116acd3698f582fcfe6b7b9fc18fd286a42bcdf007dc7a618bab958eb5c97f6082104ff723705d36744289b9885a692c01731cb45c698 \ No newline at end of file diff --git a/.hmy/extbls/5a18d4aa3e6aff4835f07588ae66be19684476d38799f63e54c6b5732fad1e86dce7458b1c295404fb54a0d61e50bb97.pass b/.hmy/extbls/5a18d4aa3e6aff4835f07588ae66be19684476d38799f63e54c6b5732fad1e86dce7458b1c295404fb54a0d61e50bb97.pass new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.hmy/extbls/7dcc035a943e29e17959dabe636efad7303d2c6f273ace457ba9dcc2fd19d3f37e70ba1cd8d082cf8ff7be2f861db48c.key b/.hmy/extbls/7dcc035a943e29e17959dabe636efad7303d2c6f273ace457ba9dcc2fd19d3f37e70ba1cd8d082cf8ff7be2f861db48c.key new file mode 100644 index 0000000000..8788231f87 --- /dev/null +++ b/.hmy/extbls/7dcc035a943e29e17959dabe636efad7303d2c6f273ace457ba9dcc2fd19d3f37e70ba1cd8d082cf8ff7be2f861db48c.key @@ -0,0 +1 @@ +eb388b59c8a4ed1d6a1713551a3404e775fdf27b5f92d302b01874e2ced1465ff9baa93eba1d3e24620d71e0557231087bbfea0c89cdb829c3868e990abaaa7595a61a13ba40d61262c5395066bccf2681f65b9f53a621a37e1b8123 \ No newline at end of file diff --git a/.hmy/extbls/7dcc035a943e29e17959dabe636efad7303d2c6f273ace457ba9dcc2fd19d3f37e70ba1cd8d082cf8ff7be2f861db48c.pass b/.hmy/extbls/7dcc035a943e29e17959dabe636efad7303d2c6f273ace457ba9dcc2fd19d3f37e70ba1cd8d082cf8ff7be2f861db48c.pass new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.hmy/extbls/81296eedba05047594385e3086e1dab52c9eb9e56f46d86f58447cccc20535d646120171961d74968d27a2ec0f8af285.key b/.hmy/extbls/81296eedba05047594385e3086e1dab52c9eb9e56f46d86f58447cccc20535d646120171961d74968d27a2ec0f8af285.key new file mode 100644 index 0000000000..86f0648745 --- /dev/null +++ b/.hmy/extbls/81296eedba05047594385e3086e1dab52c9eb9e56f46d86f58447cccc20535d646120171961d74968d27a2ec0f8af285.key @@ -0,0 +1 @@ +b64f0601353691a6bbe68658103c044ec0e075ebe2cdb8328a07269c1cf4e5574ac81fb48fb49c90cf1e0fbf0e5ed97a5806a30505104717a68d7ae341c08a7ef98bf5d4c607c236ef80d9dbfc0d3212191e0c0436b4d78890b7da68 \ No newline at end of file diff --git a/.hmy/extbls/81296eedba05047594385e3086e1dab52c9eb9e56f46d86f58447cccc20535d646120171961d74968d27a2ec0f8af285.pass b/.hmy/extbls/81296eedba05047594385e3086e1dab52c9eb9e56f46d86f58447cccc20535d646120171961d74968d27a2ec0f8af285.pass new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.hmy/extbls/89eab762e7364d6cf89f7a6c54da794f74eba2e29147992ac66adcef0f0654ef8a727710ee55ad8b532da0dd87811915.key b/.hmy/extbls/89eab762e7364d6cf89f7a6c54da794f74eba2e29147992ac66adcef0f0654ef8a727710ee55ad8b532da0dd87811915.key new file mode 100644 index 0000000000..6a86cab67d --- /dev/null +++ b/.hmy/extbls/89eab762e7364d6cf89f7a6c54da794f74eba2e29147992ac66adcef0f0654ef8a727710ee55ad8b532da0dd87811915.key @@ -0,0 +1 @@ +2ea158c2fd1d4cefcfe4dc4d987099ef93731decee88db87d007d6a5da2b87e4528abe3013814fc7183651ac1e301b2f3caa8f03071d2bd110746be8c1004dd88f22495449ae6d8c5a7cce5783a6964a4663c9319570433a68ef2f31 \ No newline at end of file diff --git a/.hmy/extbls/89eab762e7364d6cf89f7a6c54da794f74eba2e29147992ac66adcef0f0654ef8a727710ee55ad8b532da0dd87811915.pass b/.hmy/extbls/89eab762e7364d6cf89f7a6c54da794f74eba2e29147992ac66adcef0f0654ef8a727710ee55ad8b532da0dd87811915.pass new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.hmy/extbls/b0917378b179a519a5055259c4f8980cce37d58af300b00dd98b07076d3d9a3b16c4a55f84522f553872225a7b1efc0c.key b/.hmy/extbls/b0917378b179a519a5055259c4f8980cce37d58af300b00dd98b07076d3d9a3b16c4a55f84522f553872225a7b1efc0c.key new file mode 100644 index 0000000000..1f6ad8e860 --- /dev/null +++ b/.hmy/extbls/b0917378b179a519a5055259c4f8980cce37d58af300b00dd98b07076d3d9a3b16c4a55f84522f553872225a7b1efc0c.key @@ -0,0 +1 @@ +81e67ac67dc10c4d89baa7ae800d6afffbc4105766a2202ee30998669e118ce2842ae64cb8a20282d10974c1bba6ce3cdda6e2bb00124bf4ab154aeebe081a7c44ce5b010b0b069bfa37d35beccffe44209ac376b4b14885f5b3625a \ No newline at end of file diff --git a/.hmy/extbls/b0917378b179a519a5055259c4f8980cce37d58af300b00dd98b07076d3d9a3b16c4a55f84522f553872225a7b1efc0c.pass b/.hmy/extbls/b0917378b179a519a5055259c4f8980cce37d58af300b00dd98b07076d3d9a3b16c4a55f84522f553872225a7b1efc0c.pass new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.hmy/extkeystore/one17ughrllgnzx9sfa46p568k8rdmtz7qj85slc6t.key b/.hmy/extkeystore/one17ughrllgnzx9sfa46p568k8rdmtz7qj85slc6t.key new file mode 100644 index 0000000000..90157f52a5 --- /dev/null +++ b/.hmy/extkeystore/one17ughrllgnzx9sfa46p568k8rdmtz7qj85slc6t.key @@ -0,0 +1 @@ +{"address":"f71171ffe8988c5827b5d069a3d8e36ed62f0247","crypto":{"cipher":"aes-128-ctr","ciphertext":"5c7590efba91015b67234ffa063dc56e0460af0b68fa0265ac46d46a3c28448e","cipherparams":{"iv":"e8927574296604528514d7dabc6895ad"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"d2573eadafda1a530300683d610ab67465fded0e33d0a9dff101481dd4e9c043"},"mac":"0666235e715fc0c52b0cc480891b2c7514e4731d99fe93bee3d2678679f0de19"},"id":"94a50e8b-5973-4989-aaac-420cdb243863","version":3} \ No newline at end of file diff --git a/.hmy/extkeystore/one19aw2wcr5y4lxeuwt0ajgt5aw3a3qkjdgg67ygj.key b/.hmy/extkeystore/one19aw2wcr5y4lxeuwt0ajgt5aw3a3qkjdgg67ygj.key new file mode 100644 index 0000000000..2699190f5e --- /dev/null +++ b/.hmy/extkeystore/one19aw2wcr5y4lxeuwt0ajgt5aw3a3qkjdgg67ygj.key @@ -0,0 +1 @@ +{"address":"2f5ca76074257e6cf1cb7f6485d3ae8f620b49a8","crypto":{"cipher":"aes-128-ctr","ciphertext":"0f4905db05ba9cf8d99e0f090dc36bc3e15b2fe6e0247b4800c0d092bcd61a6d","cipherparams":{"iv":"fa9e222a88e5954274195fce7d2e3cc2"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"f6b32095d86b01d85764f099f6f82e2a9ae141eaf058025047e55f9ba90d0ac5"},"mac":"05f3de89292e8b16ee63181e762feb0b9fd46856b8370ebbe2c9eb806b3d26b7"},"id":"5342b382-696e-46d3-9894-272a6da3f5b3","version":3} \ No newline at end of file diff --git a/.hmy/extkeystore/one19zzwsxr0uf2fe34y8qkadek2v0eh6h5mg2deg6.key b/.hmy/extkeystore/one19zzwsxr0uf2fe34y8qkadek2v0eh6h5mg2deg6.key new file mode 100644 index 0000000000..7a697ae7a4 --- /dev/null +++ b/.hmy/extkeystore/one19zzwsxr0uf2fe34y8qkadek2v0eh6h5mg2deg6.key @@ -0,0 +1 @@ +{"address":"2884e8186fe2549cc6a4382dd6e6ca63f37d5e9b","crypto":{"cipher":"aes-128-ctr","ciphertext":"7308d5b4904e912d0f312208b9e8a95eee04d9cacf80f2c89dd802bdcd6b9cb5","cipherparams":{"iv":"ff07874ea030a9c3c9a0c8ce4d87f06a"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"972333a39a53e1dab69d50d2cd9a97e19595896e2b79e88c9ffd81b66fe93bea"},"mac":"828fba7c39d73996a5b37d399013b7659c4a9cebc40d6e1837e6253a01b1c4f1"},"id":"76c4c1ba-cd6d-4613-a89e-f9faaa7d130d","version":3} \ No newline at end of file diff --git a/.hmy/extkeystore/one1auqndgthqu5lznsn7tuma8s5333cq0y07cwc6x.key b/.hmy/extkeystore/one1auqndgthqu5lznsn7tuma8s5333cq0y07cwc6x.key new file mode 100644 index 0000000000..c64fdf2efa --- /dev/null +++ b/.hmy/extkeystore/one1auqndgthqu5lznsn7tuma8s5333cq0y07cwc6x.key @@ -0,0 +1 @@ +{"address":"ef0136a1770729f14e13f2f9be9e148c63803c8f","crypto":{"cipher":"aes-128-ctr","ciphertext":"367d95f415cf795aa7498025efb7b44ef31a698f3820bb77e3cd447d57dcfc80","cipherparams":{"iv":"206f60b938aad4f2a6763ba2199ce737"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"02c7e01ef989794d695d7ee87a5413fbea79751a8e725f529ac39e6b611da0ab"},"mac":"65e93d97f3a89bc45da4b1bcf8c38ef3b0dbee457c91207bf2c9ff787aa12ebe"},"id":"9b6fb83f-90b2-4ae6-a24a-5b13436db30f","version":3} \ No newline at end of file diff --git a/.hmy/extkeystore/one1eenp9ujcrmyaq22ef6jrpry2k97tjz4xs6ppcf.key b/.hmy/extkeystore/one1eenp9ujcrmyaq22ef6jrpry2k97tjz4xs6ppcf.key new file mode 100644 index 0000000000..92a8a23fe7 --- /dev/null +++ b/.hmy/extkeystore/one1eenp9ujcrmyaq22ef6jrpry2k97tjz4xs6ppcf.key @@ -0,0 +1 @@ +{"address":"ce6612f2581ec9d029594ea4308c8ab17cb90aa6","crypto":{"cipher":"aes-128-ctr","ciphertext":"7c4ba34f62d62734df18c383bafc03c6a8483f99ebbb1103517a7d6060c68c10","cipherparams":{"iv":"a300902fe8cc9bb4bfc686eb77b6bf26"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"693073fcc4cdb230aae22796094a1f8b17c298f78b3cabff3f8c78b81da8a222"},"mac":"624ed38319f3b46405a7d5e831fdb60dbe4d7f66f16ff9cc67bb0919ab3862e9"},"id":"19af9349-6c8d-4dd3-8053-68b03a646d59","version":3} \ No newline at end of file diff --git a/.hmy/extkeystore/one1lctumupg2y009pjmnhnmn4nqjk0zf0dspjanf7.key b/.hmy/extkeystore/one1lctumupg2y009pjmnhnmn4nqjk0zf0dspjanf7.key new file mode 100644 index 0000000000..c849d243ca --- /dev/null +++ b/.hmy/extkeystore/one1lctumupg2y009pjmnhnmn4nqjk0zf0dspjanf7.key @@ -0,0 +1 @@ +{"address":"fe17cdf028511ef2865b9de7b9d660959e24bdb0","crypto":{"cipher":"aes-128-ctr","ciphertext":"5cb69fbe4f2aaa0500d4b197e8411fa4d9fa75ff4b4bf067760c1ff79e57a0b6","cipherparams":{"iv":"5d1e9bda47aae3385d31bf036ba6db67"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"779fba35725c4711bc5772102234f3b718c46d751bf106153c06e61984e8e827"},"mac":"c1de2e2e1e5aebd5a083aec65e18eec21c67d257799a8d6402c3878045678ea8"},"id":"c2e234e2-bc52-4c38-8f82-6a6cfb65f342","version":3} \ No newline at end of file diff --git a/Makefile b/Makefile index 4c764411ec..052d6ad420 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ RPMBUILD=$(HOME)/rpmbuild DEBBUILD=$(HOME)/debbuild SHELL := bash -.PHONY: all help libs exe race trace-pointer debug debug-kill test test-go test-api test-api-attach linux_static deb_init deb_build deb debpub_dev debpub_prod rpm_init rpm_build rpm rpmpub_dev rpmpub_prod clean distclean docker +.PHONY: all help libs exe race trace-pointer debug debug-ext debug-kill test test-go test-api test-api-attach linux_static deb_init deb_build deb debpub_dev debpub_prod rpm_init rpm_build rpm rpmpub_dev rpmpub_prod clean distclean docker all: libs bash ./scripts/go_executable_build.sh -S @@ -23,8 +23,9 @@ help: @echo "exe - build the harmony binary & bootnode" @echo "race - build the harmony binary & bootnode with race condition checks" @echo "trace-pointer - build the harmony binary & bootnode with pointer analysis" - @echo "debug - start a localnet with 2 shards (s0 rpc endpoint = localhost:9599; s1 rpc endpoint = localhost:9598)" + @echo "debug - start a localnet with 2 shards (s0 rpc endpoint = localhost:9700; s1 rpc endpoint = localhost:9800)" @echo "debug-kill - force kill the localnet" + @echo "debug-ext - start a localnet with 2 shards and external (s0 rpc endpoint = localhost:9598; s1 rpc endpoint = localhost:9596)" @echo "clean - remove node files & logs created by localnet" @echo "distclean - remove node files & logs created by localnet, and all libs" @echo "test - run the entire test suite (go test & Node API test)" @@ -60,6 +61,9 @@ debug: debug-kill: bash ./test/kill_node.sh +debug-ext: + bash ./test/debug-external.sh + clean: rm -rf ./tmp_log* rm -rf ./.dht* diff --git a/internal/configs/sharding/localnet.go b/internal/configs/sharding/localnet.go index 2d04bb6495..a31c66f0da 100644 --- a/internal/configs/sharding/localnet.go +++ b/internal/configs/sharding/localnet.go @@ -23,8 +23,8 @@ const ( localnetV1Epoch = 1 localnetEpochBlock1 = 5 - localnetBlocksPerEpoch = 5 - localnetBlocksPerEpochV2 = 10 + localnetBlocksPerEpoch = 64 + localnetBlocksPerEpochV2 = 64 localnetVdfDifficulty = 5000 // This takes about 10s to finish the vdf ) diff --git a/test/build-localnet-validator.sh b/test/build-localnet-validator.sh new file mode 100644 index 0000000000..bffc7b94a9 --- /dev/null +++ b/test/build-localnet-validator.sh @@ -0,0 +1,70 @@ +#! /bin/bash + +echo "Make sure the validator account are imported" +hmy keys import-ks .hmy/extkeystore/one17ughrllgnzx9sfa46p568k8rdmtz7qj85slc6t.key 2> /dev/null +hmy keys import-ks .hmy/extkeystore/one1auqndgthqu5lznsn7tuma8s5333cq0y07cwc6x.key 2> /dev/null +hmy keys import-ks .hmy/extkeystore/one19aw2wcr5y4lxeuwt0ajgt5aw3a3qkjdgg67ygj.key 2> /dev/null +hmy keys import-ks .hmy/extkeystore/one1eenp9ujcrmyaq22ef6jrpry2k97tjz4xs6ppcf.key 2> /dev/null +hmy keys import-ks .hmy/extkeystore/one19zzwsxr0uf2fe34y8qkadek2v0eh6h5mg2deg6.key 2> /dev/null +hmy keys import-ks .hmy/extkeystore/one1lctumupg2y009pjmnhnmn4nqjk0zf0dspjanf7.key 2> /dev/null + +echo "Let's fund all the validator account" +hmy --node=http://127.0.0.1:9500 transfer --from one1zksj3evekayy90xt4psrz8h6j2v3hla4qwz4ur --to one17ughrllgnzx9sfa46p568k8rdmtz7qj85slc6t --from-shard 0 --to-shard 0 --amount 110000 +hmy --node=http://127.0.0.1:9500 transfer --from one1zksj3evekayy90xt4psrz8h6j2v3hla4qwz4ur --to one1auqndgthqu5lznsn7tuma8s5333cq0y07cwc6x --from-shard 0 --to-shard 0 --amount 110000 +hmy --node=http://127.0.0.1:9500 transfer --from one1zksj3evekayy90xt4psrz8h6j2v3hla4qwz4ur --to one19aw2wcr5y4lxeuwt0ajgt5aw3a3qkjdgg67ygj --from-shard 0 --to-shard 0 --amount 110000 +hmy --node=http://127.0.0.1:9500 transfer --from one1zksj3evekayy90xt4psrz8h6j2v3hla4qwz4ur --to one1eenp9ujcrmyaq22ef6jrpry2k97tjz4xs6ppcf --from-shard 0 --to-shard 0 --amount 110000 +hmy --node=http://127.0.0.1:9500 transfer --from one1zksj3evekayy90xt4psrz8h6j2v3hla4qwz4ur --to one19zzwsxr0uf2fe34y8qkadek2v0eh6h5mg2deg6 --from-shard 0 --to-shard 0 --amount 110000 +hmy --node=http://127.0.0.1:9500 transfer --from one1zksj3evekayy90xt4psrz8h6j2v3hla4qwz4ur --to one1lctumupg2y009pjmnhnmn4nqjk0zf0dspjanf7 --from-shard 0 --to-shard 0 --amount 110000 + + +#wait for epoch 2 +epoch=$(hmy blockchain latest-headers --node="http://localhost:9500" | jq -r '.["result"]["beacon-chain-header"]["epoch"]') +while (( epoch < 2 )); do + echo "Not yet on epoch 2 .. waiting 30s" + epoch=$(hmy blockchain latest-headers --node="http://localhost:9500" | jq -r '.["result"]["beacon-chain-header"]["epoch"]') + sleep 30 +done + +echo "Now in epoch 2, we'll create the external validators" + +hmy --node="http://localhost:9500" staking create-validator \ + --validator-addr one17ughrllgnzx9sfa46p568k8rdmtz7qj85slc6t --amount 10000 \ + --bls-pubkeys 4f41a37a3a8d0695dd6edcc58142c6b7d98e74da5c90e79b587b3b960b6a4f5e048e6d8b8a000d77a478d44cd640270c,7dcc035a943e29e17959dabe636efad7303d2c6f273ace457ba9dcc2fd19d3f37e70ba1cd8d082cf8ff7be2f861db48c \ + --name "s0-localnet-validator1" --identity "validator1" --details "validator1" \ + --security-contact "localnet" --website "localnet.one" \ + --max-change-rate 0.1 --max-rate 0.1 --rate 0.1 \ + --max-total-delegation 100000000 --min-self-delegation 10000 --bls-pubkeys-dir .hmy/extbls/ + +hmy --node="http://localhost:9500" staking create-validator \ + --validator-addr one1auqndgthqu5lznsn7tuma8s5333cq0y07cwc6x --amount 10000 \ + --bls-pubkeys b0917378b179a519a5055259c4f8980cce37d58af300b00dd98b07076d3d9a3b16c4a55f84522f553872225a7b1efc0c \ + --name "s0-localnet-validator2" --identity "validator2" --details "validator2" \ + --security-contact "localnet" --website "localnet.one" \ + --max-change-rate 0.1 --max-rate 0.1 --rate 0.1 \ + --max-total-delegation 100000000 --min-self-delegation 10000 --bls-pubkeys-dir .hmy/extbls/ + +hmy --node="http://localhost:9500" staking create-validator \ + --validator-addr one19aw2wcr5y4lxeuwt0ajgt5aw3a3qkjdgg67ygj --amount 10000 \ + --bls-pubkeys 5a18d4aa3e6aff4835f07588ae66be19684476d38799f63e54c6b5732fad1e86dce7458b1c295404fb54a0d61e50bb97,81296eedba05047594385e3086e1dab52c9eb9e56f46d86f58447cccc20535d646120171961d74968d27a2ec0f8af285 \ + --name "s1-localnet-validator3" --identity "validator3" --details "validator3" \ + --security-contact "localnet" --website "localnet.one" \ + --max-change-rate 0.1 --max-rate 0.1 --rate 0.1 \ + --max-total-delegation 100000000 --min-self-delegation 10000 --bls-pubkeys-dir .hmy/extbls/ + +hmy --node="http://localhost:9500" staking create-validator \ + --validator-addr one1eenp9ujcrmyaq22ef6jrpry2k97tjz4xs6ppcf --amount 10000 \ + --bls-pubkeys 89eab762e7364d6cf89f7a6c54da794f74eba2e29147992ac66adcef0f0654ef8a727710ee55ad8b532da0dd87811915 \ + --name "s1-localnet-validator4" --identity "validator4" --details "validator4" \ + --security-contact "localnet" --website "localnet.one" \ + --max-change-rate 0.1 --max-rate 0.1 --rate 0.1 \ + --max-total-delegation 100000000 --min-self-delegation 10000 --bls-pubkeys-dir .hmy/extbls/ + + +echo "validator created" +echo '''check their information +hmy blockchain validator information one17ughrllgnzx9sfa46p568k8rdmtz7qj85slc6t --node="http://localhost:9500" +hmy blockchain validator information one1auqndgthqu5lznsn7tuma8s5333cq0y07cwc6x --node="http://localhost:9500" +hmy blockchain validator information one19aw2wcr5y4lxeuwt0ajgt5aw3a3qkjdgg67ygj --node="http://localhost:9500" +hmy blockchain validator information one1eenp9ujcrmyaq22ef6jrpry2k97tjz4xs6ppcf --node="http://localhost:9500" +''' + diff --git a/test/configs/local-resharding-with-external.txt b/test/configs/local-resharding-with-external.txt new file mode 100644 index 0000000000..318a08b9bc --- /dev/null +++ b/test/configs/local-resharding-with-external.txt @@ -0,0 +1,37 @@ +# shard 0 +# internal node +127.0.0.1 9000 validator .hmy/65f55eb3052f9e9f632b2923be594ba77c55543f5c58ee1454b9cfd658d25e06373b0f7d42a19c84768139ea294f6204.key +127.0.0.1 9002 validator .hmy/02c8ff0b88f313717bc3a627d2f8bb172ba3ad3bb9ba3ecb8eed4b7c878653d3d4faf769876c528b73f343967f74a917.key +127.0.0.1 9004 validator .hmy/e751ec995defe4931273aaebcb2cd14bf37e629c554a57d3f334c37881a34a6188a93e76113c55ef3481da23b7d7ab09.key +127.0.0.1 9006 validator .hmy/2d61379e44a772e5757e27ee2b3874254f56073e6bd226eb8b160371cc3c18b8c4977bd3dcb71fd57dc62bf0e143fd08.key +127.0.0.1 9008 validator .hmy/86dc2fdc2ceec18f6923b99fd86a68405c132e1005cf1df72dca75db0adfaeb53d201d66af37916d61f079f34f21fb96.key +127.0.0.1 9010 validator .hmy/95117937cd8c09acd2dfae847d74041a67834ea88662a7cbed1e170350bc329e53db151e5a0ef3e712e35287ae954818.key +# external node +127.0.0.1 9014 external .hmy/extbls/4f41a37a3a8d0695dd6edcc58142c6b7d98e74da5c90e79b587b3b960b6a4f5e048e6d8b8a000d77a478d44cd640270c.key +127.0.0.1 9016 external .hmy/extbls/7dcc035a943e29e17959dabe636efad7303d2c6f273ace457ba9dcc2fd19d3f37e70ba1cd8d082cf8ff7be2f861db48c.key +127.0.0.1 9018 external .hmy/extbls/b0917378b179a519a5055259c4f8980cce37d58af300b00dd98b07076d3d9a3b16c4a55f84522f553872225a7b1efc0c.key +# fn node +127.0.0.1 9050 validator .hmy/52ecce5f64db21cbe374c9268188f5d2cdd5bec1a3112276a350349860e35fb81f8cfe447a311e0550d961cf25cb988d.key +127.0.0.1 9052 validator .hmy/678ec9670899bf6af85b877058bea4fc1301a5a3a376987e826e3ca150b80e3eaadffedad0fedfa111576fa76ded980c.key +#127.0.0.1 9054 validator .hmy/16513c487a6bb76f37219f3c2927a4f281f9dd3fd6ed2e3a64e500de6545cf391dd973cc228d24f9bd01efe94912e714.key +# explorer node +127.0.0.1 9098 explorer null 0 + +# shard 1 +# internal node +127.0.0.1 9100 validator .hmy/40379eed79ed82bebfb4310894fd33b6a3f8413a78dc4d43b98d0adc9ef69f3285df05eaab9f2ce5f7227f8cb920e809.key +127.0.0.1 9102 validator .hmy/ee2474f93cba9241562efc7475ac2721ab0899edf8f7f115a656c0c1f9ef8203add678064878d174bb478fa2e6630502.key +127.0.0.1 9104 validator .hmy/776f3b8704f4e1092a302a60e84f81e476c212d6f458092b696df420ea19ff84a6179e8e23d090b9297dc041600bc100.key +127.0.0.1 9106 validator .hmy/c4e4708b6cf2a2ceeb59981677e9821eebafc5cf483fb5364a28fa604cc0ce69beeed40f3f03815c9e196fdaec5f1097.key +127.0.0.1 9108 validator .hmy/49d15743b36334399f9985feb0753430a2b287b2d68b84495bbb15381854cbf01bca9d1d9f4c9c8f18509b2bfa6bd40f.key +127.0.0.1 9110 validator .hmy/68ae289d73332872ec8d04ac256ca0f5453c88ad392730c5741b6055bc3ec3d086ab03637713a29f459177aaa8340615.key +# external node +127.0.0.1 9114 external .hmy/extbls/5a18d4aa3e6aff4835f07588ae66be19684476d38799f63e54c6b5732fad1e86dce7458b1c295404fb54a0d61e50bb97.key +127.0.0.1 9116 external .hmy/extbls/81296eedba05047594385e3086e1dab52c9eb9e56f46d86f58447cccc20535d646120171961d74968d27a2ec0f8af285.key +127.0.0.1 9118 external .hmy/extbls/89eab762e7364d6cf89f7a6c54da794f74eba2e29147992ac66adcef0f0654ef8a727710ee55ad8b532da0dd87811915.key +# fn node +127.0.0.1 9150 validator .hmy/a547a9bf6fdde4f4934cde21473748861a3cc0fe8bbb5e57225a29f483b05b72531f002f8187675743d819c955a86100.key +127.0.0.1 9152 validator .hmy/63f479f249c59f0486fda8caa2ffb247209489dae009dfde6144ff38c370230963d360dffd318cfb26c213320e89a512.key +#127.0.0.1 9154 validator .hmy/576d3c48294e00d6be4a22b07b66a870ddee03052fe48a5abbd180222e5d5a1f8946a78d55b025de21635fd743bbad90.key +# explorer node +127.0.0.1 9096 explorer null 1 diff --git a/test/debug-external.sh b/test/debug-external.sh new file mode 100755 index 0000000000..198b01b38b --- /dev/null +++ b/test/debug-external.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +./test/kill_node.sh +rm -rf tmp_log* +rm *.rlp +rm -rf .dht* +scripts/go_executable_build.sh -S || exit 1 # dynamic builds are faster for debug iteration... +./test/deploy.sh -B -D 600000 ./test/configs/local-resharding-with-external.txt diff --git a/test/deploy.sh b/test/deploy.sh index 9ab6943110..d310a4ed1f 100755 --- a/test/deploy.sh +++ b/test/deploy.sh @@ -83,8 +83,8 @@ function launch_localnet() { # Read config for i-th node form config file IFS=' ' read -r ip port mode bls_key shard node_config <<<"${line}" args=("${base_args[@]}" --ip "${ip}" --port "${port}" --key "/tmp/${ip}-${port}.key" --db_dir "${ROOT}/db-${ip}-${port}" "--broadcast_invalid_tx=false") - if [[ -z "$ip" || -z "$port" ]]; then - echo "skip empty node" + if [[ -z "$ip" || -z "$port" || "$ip" == "#" ]]; then + echo "skip empty line or node or comment" continue fi if [[ $EXPOSEAPIS == "true" ]]; then From a1775465d7eb5db0406adc18b44e7e3116e4aaf8 Mon Sep 17 00:00:00 2001 From: Konstantin <355847+Frozen@users.noreply.github.com> Date: Wed, 29 Mar 2023 17:17:56 -0300 Subject: [PATCH 384/420] Configurable tx pool. (#4240) * AccountQueue & GlobalQueue. * Lifetime duration. * [pool] make flags configurable * [pool] use 4096 as default `GlobalSlots` * [rosetta] update default values of tx pool * [test] update value to default * PriceLimit and PriceBump. * Fix tests. * Fix price limit & bump. * Updated, fixed migrate version and tests. * Rebased. * Fix go toml version. --------- Co-authored-by: Konstantin Co-authored-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> --- cmd/harmony/config_migrations.go | 23 +++++++- cmd/harmony/config_migrations_test.go | 26 +++++++-- cmd/harmony/config_test.go | 10 ++-- cmd/harmony/default.go | 12 +++-- cmd/harmony/flags.go | 68 +++++++++++++++++++++++- cmd/harmony/flags_test.go | 54 ++++++++++++++++++- core/tx_pool.go | 31 +++++++++-- go.sum | 2 + internal/configs/harmony/harmony.go | 29 ++++++++++ internal/configs/harmony/harmony_test.go | 37 +++++++++++++ node/node.go | 15 ++++-- rosetta/infra/harmony-mainnet.conf | 6 +++ rosetta/infra/harmony-pstn.conf | 6 +++ 13 files changed, 296 insertions(+), 23 deletions(-) diff --git a/cmd/harmony/config_migrations.go b/cmd/harmony/config_migrations.go index b3da9ec2ba..84607c2d0d 100644 --- a/cmd/harmony/config_migrations.go +++ b/cmd/harmony/config_migrations.go @@ -8,7 +8,7 @@ import ( "strings" goversion "github.com/hashicorp/go-version" - "github.com/pelletier/go-toml" + "github.com/pelletier/go-toml" // TODO support go-toml/v2 "github.com/harmony-one/harmony/api/service/legacysync" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" @@ -357,6 +357,27 @@ func init() { return confTree } + migrations["2.5.13"] = func(confTree *toml.Tree) *toml.Tree { + if confTree.Get("TxPool.AccountQueue") == nil { + confTree.Set("TxPool.AccountQueue", defaultConfig.TxPool.AccountQueue) + } + if confTree.Get("TxPool.GlobalQueue") == nil { + confTree.Set("TxPool.GlobalQueue", defaultConfig.TxPool.GlobalQueue) + } + if confTree.Get("TxPool.Lifetime") == nil { + confTree.Set("TxPool.Lifetime", defaultConfig.TxPool.Lifetime.String()) + } + if confTree.Get("TxPool.PriceLimit") == nil { + confTree.Set("TxPool.PriceLimit", defaultConfig.TxPool.PriceLimit) + } + if confTree.Get("TxPool.PriceBump") == nil { + confTree.Set("TxPool.PriceBump", defaultConfig.TxPool.PriceBump) + } + + confTree.Set("Version", "2.5.14") + return confTree + } + // check that the latest version here is the same as in default.go largestKey := getNextVersion(migrations) if largestKey != tomlConfigVersion { diff --git a/cmd/harmony/config_migrations_test.go b/cmd/harmony/config_migrations_test.go index 72db5d7900..e52c7347b2 100644 --- a/cmd/harmony/config_migrations_test.go +++ b/cmd/harmony/config_migrations_test.go @@ -1,10 +1,10 @@ package main import ( - "reflect" "testing" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" + "github.com/stretchr/testify/require" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" ) @@ -72,6 +72,11 @@ Version = "1.0.2" [TxPool] BlacklistFile = "./.hmy/blacklist.txt" LocalAccountsFile = "./.hmy/locals.txt" + AccountQueue = 64 + GlobalQueue = 5120 + Lifetime = "30m" + PriceBump = 1 + PriceLimit = 100e9 [WS] Enabled = true @@ -142,6 +147,11 @@ Version = "1.0.3" [TxPool] BlacklistFile = "./.hmy/blacklist.txt" LocalAccountsFile = "./.hmy/locals.txt" + AccountQueue = 64 + GlobalQueue = 5120 + Lifetime = "30m" + PriceBump = 1 + PriceLimit = 100e9 [WS] Enabled = true @@ -224,6 +234,11 @@ Version = "1.0.4" [TxPool] BlacklistFile = "./.hmy/blacklist.txt" LocalAccountsFile = "./.hmy/locals.txt" + AccountQueue = 64 + GlobalQueue = 5120 + Lifetime = "30m" + PriceBump = 1 + PriceLimit = 100e9 [WS] Enabled = true @@ -314,6 +329,11 @@ Version = "1.0.4" BlacklistFile = "./.hmy/blacklist.txt" LocalAccountsFile = "./.hmy/locals.txt" AllowedTxsFile = "./.hmy/allowedtxs.txt" + AccountQueue = 64 + GlobalQueue = 5120 + Lifetime = "30m" + PriceBump = 1 + PriceLimit = 100e9 [WS] Enabled = true @@ -389,9 +409,7 @@ func Test_migrateConf(t *testing.T) { t.Errorf("migrateConf() error = %v, wantErr %v", err, tt.wantErr) return } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("migrateConf() = %+v, want %+v", got, tt.want) - } + require.Equal(t, tt.want, got) }) } } diff --git a/cmd/harmony/config_test.go b/cmd/harmony/config_test.go index 0d58797008..d19f5d9858 100644 --- a/cmd/harmony/config_test.go +++ b/cmd/harmony/config_test.go @@ -9,6 +9,7 @@ import ( "testing" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" + "github.com/stretchr/testify/require" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" ) @@ -86,6 +87,11 @@ Version = "1.0.4" BlacklistFile = "./.hmy/blacklist.txt" LocalAccountsFile = "./.hmy/locals.txt" AllowedTxsFile = "./.hmy/allowedtxs.txt" + AccountQueue = 64 + GlobalQueue = 5120 + Lifetime = "30m" + PriceBump = 1 + PriceLimit = 100e9 [Sync] Downloader = false @@ -136,9 +142,7 @@ Version = "1.0.4" t.Errorf("Expected config version: 1.0.4, not %v", config.Version) } config.Version = defConf.Version // Shortcut for testing, value checked above - if !reflect.DeepEqual(config, defConf) { - t.Errorf("Unexpected config \n\t%+v \n\t%+v", config, defaultConfig) - } + require.Equal(t, config, defConf) } func TestPersistConfig(t *testing.T) { diff --git a/cmd/harmony/default.go b/cmd/harmony/default.go index 95e05b29cc..2c15e123f7 100644 --- a/cmd/harmony/default.go +++ b/cmd/harmony/default.go @@ -1,11 +1,12 @@ package main import ( + "github.com/harmony-one/harmony/core" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" ) -const tomlConfigVersion = "2.5.13" +const tomlConfigVersion = "2.5.14" const ( defNetworkType = nodeconfig.Mainnet @@ -81,9 +82,14 @@ var defaultConfig = harmonyconfig.HarmonyConfig{ BlacklistFile: "./.hmy/blacklist.txt", AllowedTxsFile: "./.hmy/allowedtxs.txt", RosettaFixFile: "", - AccountSlots: 16, + AccountSlots: core.DefaultTxPoolConfig.AccountSlots, LocalAccountsFile: "./.hmy/locals.txt", - GlobalSlots: 5120, + GlobalSlots: core.DefaultTxPoolConfig.GlobalSlots, + AccountQueue: core.DefaultTxPoolConfig.AccountQueue, + GlobalQueue: core.DefaultTxPoolConfig.GlobalQueue, + Lifetime: core.DefaultTxPoolConfig.Lifetime, + PriceLimit: harmonyconfig.PriceLimit(core.DefaultTxPoolConfig.PriceLimit), + PriceBump: core.DefaultTxPoolConfig.PriceBump, }, Sync: getDefaultSyncConfig(defNetworkType), Pprof: harmonyconfig.PprofConfig{ diff --git a/cmd/harmony/flags.go b/cmd/harmony/flags.go index 8a2799ce28..9760652e03 100644 --- a/cmd/harmony/flags.go +++ b/cmd/harmony/flags.go @@ -4,6 +4,7 @@ import ( "fmt" "strconv" "strings" + "time" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" @@ -139,12 +140,17 @@ var ( txPoolFlags = []cli.Flag{ tpAccountSlotsFlag, + tpGlobalSlotsFlag, + tpAccountQueueFlag, + tpGlobalQueueFlag, + tpLifetimeFlag, rosettaFixFileFlag, tpBlacklistFileFlag, legacyTPBlacklistFileFlag, localAccountsFileFlag, allowedTxsFileFlag, - tpGlobalSlotsFlag, + tpPriceLimitFlag, + tpPriceBumpFlag, } pprofFlags = []cli.Flag{ @@ -1186,6 +1192,31 @@ var ( Usage: "maximum global number of non-executable transactions in the pool", DefValue: int(defaultConfig.TxPool.GlobalSlots), } + tpAccountQueueFlag = cli.IntFlag{ + Name: "txpool.accountqueue", + Usage: "capacity of queued transactions for account in the pool", + DefValue: int(defaultConfig.TxPool.AccountQueue), + } + tpGlobalQueueFlag = cli.IntFlag{ + Name: "txpool.globalqueue", + Usage: "global capacity for queued transactions in the pool", + DefValue: int(defaultConfig.TxPool.GlobalQueue), + } + tpLifetimeFlag = cli.StringFlag{ + Name: "txpool.lifetime", + Usage: "maximum lifetime of transactions in the pool as a golang duration string", + DefValue: defaultConfig.TxPool.Lifetime.String(), + } + tpPriceLimitFlag = cli.IntFlag{ + Name: "txpool.pricelimit", + Usage: "minimum gas price to enforce for acceptance into the pool", + DefValue: int(defaultConfig.TxPool.PriceLimit), + } + tpPriceBumpFlag = cli.IntFlag{ + Name: "txpool.pricebump", + Usage: "minimum price bump to replace an already existing transaction (nonce)", + DefValue: int(defaultConfig.TxPool.PriceLimit), + } ) func applyTxPoolFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { @@ -1206,6 +1237,20 @@ func applyTxPoolFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { } config.TxPool.GlobalSlots = uint64(value) } + if cli.IsFlagChanged(cmd, tpAccountQueueFlag) { + value := cli.GetIntFlagValue(cmd, tpAccountQueueFlag) + if value <= 0 { + panic("Must provide positive value for txpool.accountqueue") + } + config.TxPool.AccountQueue = uint64(value) + } + if cli.IsFlagChanged(cmd, tpGlobalQueueFlag) { + value := cli.GetIntFlagValue(cmd, tpGlobalQueueFlag) + if value <= 0 { + panic("Must provide positive value for txpool.globalqueue") + } + config.TxPool.GlobalQueue = uint64(value) + } if cli.IsFlagChanged(cmd, tpBlacklistFileFlag) { config.TxPool.BlacklistFile = cli.GetStringFlagValue(cmd, tpBlacklistFileFlag) } else if cli.IsFlagChanged(cmd, legacyTPBlacklistFileFlag) { @@ -1217,6 +1262,27 @@ func applyTxPoolFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { if cli.IsFlagChanged(cmd, allowedTxsFileFlag) { config.TxPool.AllowedTxsFile = cli.GetStringFlagValue(cmd, allowedTxsFileFlag) } + if cli.IsFlagChanged(cmd, tpLifetimeFlag) { + value, err := time.ParseDuration(cli.GetStringFlagValue(cmd, tpLifetimeFlag)) + if err != nil { + panic(fmt.Sprintf("Invalid value for txpool.lifetime: %v", err)) + } + config.TxPool.Lifetime = value + } + if cli.IsFlagChanged(cmd, tpPriceLimitFlag) { + value := cli.GetIntFlagValue(cmd, tpPriceLimitFlag) + if value <= 0 { + panic("Must provide positive value for txpool.pricelimit") + } + config.TxPool.PriceLimit = harmonyconfig.PriceLimit(value) + } + if cli.IsFlagChanged(cmd, tpPriceBumpFlag) { + value := cli.GetIntFlagValue(cmd, tpPriceBumpFlag) + if value <= 0 { + panic("Must provide positive value for txpool.pricebump") + } + config.TxPool.PriceBump = uint64(value) + } } // pprof flags diff --git a/cmd/harmony/flags_test.go b/cmd/harmony/flags_test.go index 2015188ed4..5338b9f71e 100644 --- a/cmd/harmony/flags_test.go +++ b/cmd/harmony/flags_test.go @@ -5,6 +5,7 @@ import ( "reflect" "strings" "testing" + "time" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" @@ -118,8 +119,13 @@ func TestHarmonyFlags(t *testing.T) { AllowedTxsFile: "./.hmy/allowedtxs.txt", RosettaFixFile: "", AccountSlots: 16, - GlobalSlots: 5120, + GlobalSlots: 4096, LocalAccountsFile: "./.hmy/locals.txt", + AccountQueue: 64, + GlobalQueue: 5120, + Lifetime: 30 * time.Minute, + PriceLimit: 100e9, + PriceBump: 1, }, Pprof: harmonyconfig.PprofConfig{ Enabled: false, @@ -1005,6 +1011,11 @@ func TestTxPoolFlags(t *testing.T) { AccountSlots: defaultConfig.TxPool.AccountSlots, LocalAccountsFile: defaultConfig.TxPool.LocalAccountsFile, GlobalSlots: defaultConfig.TxPool.GlobalSlots, + AccountQueue: defaultConfig.TxPool.AccountQueue, + GlobalQueue: defaultConfig.TxPool.GlobalQueue, + Lifetime: defaultConfig.TxPool.Lifetime, + PriceLimit: 100e9, + PriceBump: 1, }, }, { @@ -1015,7 +1026,12 @@ func TestTxPoolFlags(t *testing.T) { RosettaFixFile: "rosettafix.file", AccountSlots: defaultConfig.TxPool.AccountSlots, GlobalSlots: defaultConfig.TxPool.GlobalSlots, + AccountQueue: defaultConfig.TxPool.AccountQueue, + GlobalQueue: defaultConfig.TxPool.GlobalQueue, + Lifetime: defaultConfig.TxPool.Lifetime, LocalAccountsFile: defaultConfig.TxPool.LocalAccountsFile, + PriceLimit: 100e9, + PriceBump: 1, }, }, { @@ -1026,7 +1042,12 @@ func TestTxPoolFlags(t *testing.T) { AllowedTxsFile: defaultConfig.TxPool.AllowedTxsFile, AccountSlots: defaultConfig.TxPool.AccountSlots, GlobalSlots: defaultConfig.TxPool.GlobalSlots, + AccountQueue: defaultConfig.TxPool.AccountQueue, + GlobalQueue: defaultConfig.TxPool.GlobalQueue, + Lifetime: defaultConfig.TxPool.Lifetime, LocalAccountsFile: defaultConfig.TxPool.LocalAccountsFile, + PriceLimit: 100e9, + PriceBump: 1, }, }, { @@ -1038,6 +1059,11 @@ func TestTxPoolFlags(t *testing.T) { RosettaFixFile: "rosettafix.file", LocalAccountsFile: defaultConfig.TxPool.LocalAccountsFile, GlobalSlots: defaultConfig.TxPool.GlobalSlots, + AccountQueue: defaultConfig.TxPool.AccountQueue, + GlobalQueue: defaultConfig.TxPool.GlobalQueue, + Lifetime: defaultConfig.TxPool.Lifetime, + PriceLimit: 100e9, + PriceBump: 1, }, }, { @@ -1049,6 +1075,11 @@ func TestTxPoolFlags(t *testing.T) { AccountSlots: defaultConfig.TxPool.AccountSlots, LocalAccountsFile: "locals.txt", GlobalSlots: defaultConfig.TxPool.GlobalSlots, + AccountQueue: defaultConfig.TxPool.AccountQueue, + GlobalQueue: defaultConfig.TxPool.GlobalQueue, + Lifetime: defaultConfig.TxPool.Lifetime, + PriceLimit: 100e9, + PriceBump: 1, }, }, { @@ -1060,6 +1091,27 @@ func TestTxPoolFlags(t *testing.T) { AccountSlots: defaultConfig.TxPool.AccountSlots, LocalAccountsFile: defaultConfig.TxPool.LocalAccountsFile, GlobalSlots: 10240, + AccountQueue: defaultConfig.TxPool.AccountQueue, + GlobalQueue: defaultConfig.TxPool.GlobalQueue, + Lifetime: defaultConfig.TxPool.Lifetime, + PriceLimit: 100e9, + PriceBump: 1, + }, + }, + { + args: []string{"--txpool.accountqueue", "128", "--txpool.globalqueue", "10240", "--txpool.lifetime", "15m", "--txpool.pricelimit", "100", "--txpool.pricebump", "2"}, + expConfig: harmonyconfig.TxPoolConfig{ + BlacklistFile: defaultConfig.TxPool.BlacklistFile, + AllowedTxsFile: defaultConfig.TxPool.AllowedTxsFile, + RosettaFixFile: defaultConfig.TxPool.RosettaFixFile, + AccountSlots: defaultConfig.TxPool.AccountSlots, + LocalAccountsFile: defaultConfig.TxPool.LocalAccountsFile, + GlobalSlots: defaultConfig.TxPool.GlobalSlots, + AccountQueue: 128, + GlobalQueue: 10240, + Lifetime: 15 * time.Minute, + PriceLimit: 100, + PriceBump: 2, }, }, } diff --git a/core/tx_pool.go b/core/tx_pool.go index 2efaf3be01..bd062b6f4c 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -185,12 +185,12 @@ var DefaultTxPoolConfig = TxPoolConfig{ PriceLimit: 100e9, // 100 Gwei/Nano PriceBump: 1, // PriceBump is percent, 1% is enough - AccountSlots: 16, - GlobalSlots: 4096, - AccountQueue: 64, - GlobalQueue: 1024, + AccountSlots: 16, // --txpool.accountslots + GlobalSlots: 4096, // --txpool.globalslots + AccountQueue: 64, // --txpool.accountqueue + GlobalQueue: 5120, // --txpool.globalqueue - Lifetime: 30 * time.Minute, + Lifetime: 30 * time.Minute, // --txpool.lifetime Blacklist: map[common.Address]struct{}{}, AllowedTxs: map[common.Address]AllowedTxData{}, @@ -243,6 +243,27 @@ func (config *TxPoolConfig) sanitize() TxPoolConfig { Msg("Sanitizing invalid txpool global slots") conf.GlobalSlots = DefaultTxPoolConfig.GlobalSlots } + if conf.AccountQueue == 0 { + utils.Logger().Warn(). + Uint64("provided", conf.AccountQueue). + Uint64("updated", DefaultTxPoolConfig.AccountQueue). + Msg("Sanitizing invalid txpool account queue") + conf.AccountQueue = DefaultTxPoolConfig.AccountQueue + } + if conf.GlobalQueue == 0 { + utils.Logger().Warn(). + Uint64("provided", conf.GlobalQueue). + Uint64("updated", DefaultTxPoolConfig.GlobalQueue). + Msg("Sanitizing invalid txpool account queue") + conf.GlobalQueue = DefaultTxPoolConfig.GlobalQueue + } + if conf.Lifetime == 0 { + utils.Logger().Warn(). + Dur("provided", conf.Lifetime). + Dur("updated", DefaultTxPoolConfig.Lifetime). + Msg("Sanitizing invalid txpool lifetime") + conf.Lifetime = DefaultTxPoolConfig.Lifetime + } return conf } diff --git a/go.sum b/go.sum index 2ebb5e94a2..46f3d441b0 100644 --- a/go.sum +++ b/go.sum @@ -774,6 +774,8 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= diff --git a/internal/configs/harmony/harmony.go b/internal/configs/harmony/harmony.go index d4e8df4b00..32161d1deb 100644 --- a/internal/configs/harmony/harmony.go +++ b/internal/configs/harmony/harmony.go @@ -1,6 +1,7 @@ package harmony import ( + "fmt" "reflect" "strings" "time" @@ -181,8 +182,13 @@ type TxPoolConfig struct { AllowedTxsFile string RosettaFixFile string AccountSlots uint64 + AccountQueue uint64 + GlobalQueue uint64 LocalAccountsFile string GlobalSlots uint64 + Lifetime time.Duration + PriceLimit PriceLimit + PriceBump uint64 } type PprofConfig struct { @@ -318,3 +324,26 @@ type StagedSyncConfig struct { UseMemDB bool // it uses memory by default. set it to false to use disk LogProgress bool // log the full sync progress in console } + +type PriceLimit int64 + +func (s *PriceLimit) UnmarshalTOML(data interface{}) error { + switch v := data.(type) { + case float64: + *s = PriceLimit(v) + case int64: + *s = PriceLimit(v) + case PriceLimit: + *s = v + default: + return fmt.Errorf("PriceLimit.UnmarshalTOML: %T", data) + } + return nil +} + +func (s PriceLimit) MarshalTOML() ([]byte, error) { + if s > 1_000_000_000 { + return []byte(fmt.Sprintf("%de9", s/1_000_000_000)), nil + } + return []byte(fmt.Sprintf("%d", s)), nil +} diff --git a/internal/configs/harmony/harmony_test.go b/internal/configs/harmony/harmony_test.go index fef7cac9df..f51e2ef2f7 100644 --- a/internal/configs/harmony/harmony_test.go +++ b/internal/configs/harmony/harmony_test.go @@ -6,7 +6,9 @@ import ( "time" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + "github.com/pelletier/go-toml" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestToRPCServerConfig(t *testing.T) { @@ -79,3 +81,38 @@ func TestToRPCServerConfig(t *testing.T) { }) } } + +var data = `big = 100e9 +small = 100 +zero = 0 +` + +func TestPriceLimit_UnmarshalTOML(t *testing.T) { + type V struct { + Big PriceLimit `toml:"big"` + Small PriceLimit `toml:"small"` + Zero PriceLimit `toml:"zero"` + } + var v V + require.NoError(t, toml.Unmarshal([]byte(data), &v)) + + require.Equal(t, PriceLimit(100e9), v.Big) + require.Equal(t, PriceLimit(100), v.Small) + require.Equal(t, PriceLimit(0), v.Zero) +} + +func TestPriceLimit_MarshalTOML(t *testing.T) { + type V struct { + Big PriceLimit `toml:"big"` + Small PriceLimit `toml:"small"` + Zero PriceLimit `toml:"zero"` + } + v := V{ + Big: PriceLimit(100e9), + Small: PriceLimit(100), + Zero: PriceLimit(0), + } + e, err := toml.Marshal(v) + require.NoError(t, err) + require.Equal(t, data, string(e)) +} diff --git a/node/node.go b/node/node.go index 13988e2ad3..b41163ac47 100644 --- a/node/node.go +++ b/node/node.go @@ -1084,15 +1084,20 @@ func New( node.BeaconBlockChannel = make(chan *types.Block) txPoolConfig := core.DefaultTxPoolConfig - // Temporarily not updating other networks to make the rpc tests pass - if node.NodeConfig.GetNetworkType() != nodeconfig.Mainnet && node.NodeConfig.GetNetworkType() != nodeconfig.Testnet { - txPoolConfig.PriceLimit = 1e9 - txPoolConfig.PriceBump = 10 - } if harmonyconfig != nil { txPoolConfig.AccountSlots = harmonyconfig.TxPool.AccountSlots txPoolConfig.GlobalSlots = harmonyconfig.TxPool.GlobalSlots txPoolConfig.Locals = append(txPoolConfig.Locals, localAccounts...) + txPoolConfig.AccountQueue = harmonyconfig.TxPool.AccountQueue + txPoolConfig.GlobalQueue = harmonyconfig.TxPool.GlobalQueue + txPoolConfig.Lifetime = harmonyconfig.TxPool.Lifetime + txPoolConfig.PriceLimit = uint64(harmonyconfig.TxPool.PriceLimit) + txPoolConfig.PriceBump = harmonyconfig.TxPool.PriceBump + } + // Temporarily not updating other networks to make the rpc tests pass + if node.NodeConfig.GetNetworkType() != nodeconfig.Mainnet && node.NodeConfig.GetNetworkType() != nodeconfig.Testnet { + txPoolConfig.PriceLimit = 1e9 + txPoolConfig.PriceBump = 10 } txPoolConfig.Blacklist = blacklist diff --git a/rosetta/infra/harmony-mainnet.conf b/rosetta/infra/harmony-mainnet.conf index 8d51609cb1..6deea22942 100644 --- a/rosetta/infra/harmony-mainnet.conf +++ b/rosetta/infra/harmony-mainnet.conf @@ -133,6 +133,12 @@ Version = "2.5.13" GlobalSlots = 5120 LocalAccountsFile = "./.hmy/locals.txt" RosettaFixFile = "./rosetta_local_fix.csv" + GlobalSlots = 4096 + GlobalQueue = 5120 + AccountQueue = 64 + Lifetime = "30m" + PriceBump = 1 + PriceLimit = 100e9 [WS] AuthPort = 9801 diff --git a/rosetta/infra/harmony-pstn.conf b/rosetta/infra/harmony-pstn.conf index 1bb865c1a5..49dbe01c10 100644 --- a/rosetta/infra/harmony-pstn.conf +++ b/rosetta/infra/harmony-pstn.conf @@ -133,6 +133,12 @@ Version = "2.5.13" GlobalSlots = 5120 LocalAccountsFile = "./.hmy/locals.txt" RosettaFixFile = "" + GlobalSlots = 4096 + GlobalQueue = 5120 + AccountQueue = 64 + Lifetime = "30m" + PriceBump = 1 + PriceLimit = 100e9 [WS] AuthPort = 9801 From d21dc7f200dd6bf3199e4ec34ac6f7a1c894ea50 Mon Sep 17 00:00:00 2001 From: Gheis Mohammadi Date: Fri, 31 Mar 2023 09:28:07 +0800 Subject: [PATCH 385/420] Upgrade rawdb and statedb codes to add the latest functionalities of ethdb (#4374) * added bloom filter * upgrade rawdb and statedb * change var name and remove extra comments * return back fake storage in case if we need it for test later * add the previous change back * remove some extra entries from go.mod * fix WritePreimages to use batch * mark unused functions which are ported over from eth --------- Co-authored-by: Casey Gardiner <117784577+ONECasey@users.noreply.github.com> --- accounts/abi/bind/util_test.go | 18 +- accounts/external/backend.go | 7 +- api/service/explorer/interface.go | 6 +- cmd/harmony/dumpdb.go | 13 +- contracts/Puzzle.go | 44 +- core/blockchain_impl.go | 36 +- core/blockchain_pruner.go | 5 + core/genesis.go | 2 +- core/rawdb/accessors_chain.go | 529 ++++++++++- core/rawdb/accessors_chain_test.go | 6 +- core/rawdb/accessors_indexes.go | 76 +- core/rawdb/accessors_metadata.go | 162 +++- core/rawdb/accessors_offchain.go | 10 +- core/rawdb/accessors_snapshot.go | 210 +++++ core/rawdb/accessors_state.go | 95 ++ core/rawdb/accessors_sync.go | 80 ++ core/rawdb/accessors_trie.go | 263 ++++++ core/rawdb/ancient_scheme.go | 55 ++ core/rawdb/ancient_utils.go | 91 ++ core/rawdb/chain_iterator.go | 358 ++++++++ core/rawdb/database.go | 464 ++++++++++ core/rawdb/database_test.go | 17 + core/rawdb/databases_64bit.go | 37 + core/rawdb/databases_non64bit.go | 34 + core/rawdb/interfaces.go | 4 +- core/rawdb/key_length_iterator.go | 47 + core/rawdb/key_length_iterator_test.go | 60 ++ core/rawdb/schema.go | 147 +++- core/rawdb/table.go | 313 +++++++ core/rawdb/table_test.go | 128 +++ core/rawdb/testdata/stored_receipts.bin | Bin 0 -> 99991 bytes core/staking_verifier_test.go | 2 +- core/state/access_list.go | 136 +++ core/state/database.go | 137 ++- core/state/dump.go | 82 +- core/state/iterator.go | 155 ++++ core/state/iterator_test.go | 142 +++ core/state/journal.go | 75 +- core/state/managed_state_test.go | 2 +- core/state/metrics.go | 30 + core/state/prefeth.go | 11 +- core/state/state_object.go | 261 ++++-- core/state/state_object_test.go | 46 + core/state/state_test.go | 13 +- core/state/statedb.go | 814 ++++++++++++----- core/state/statedb_test.go | 831 +++++++----------- core/state/sync.go | 56 ++ core/state/tikv_clean.go | 13 +- core/state/transient_storage.go | 55 ++ core/state/trie_prefetcher.go | 354 ++++++++ core/state/trie_prefetcher_test.go | 110 +++ core/state_processor.go | 4 +- core/tx_pool.go | 6 +- core/tx_pool_test.go | 28 +- core/types/bloom9_test.go | 8 +- core/vm/contracts_write_test.go | 4 +- core/vm/gas_table_test.go | 2 +- core/vm/logger_test.go | 3 +- core/vm/runtime/runtime.go | 4 +- core/vm/runtime/runtime_test.go | 4 +- go.mod | 65 +- go.sum | 755 +++++++++++++--- hmy/blockchain.go | 4 +- hmy/tracer.go | 6 +- internal/chain/engine_test.go | 2 +- internal/shardchain/dbfactory.go | 4 +- internal/shardchain/leveldb_shard/shard.go | 10 +- internal/tikv/common/tikv_store.go | 7 +- internal/tikv/prefix/prefix_database.go | 9 + internal/tikv/remote/remote_database.go | 8 + .../statedb_cache/statedb_cache_database.go | 8 + rpc/blockchain.go | 5 +- staking/slash/double-sign_test.go | 2 +- test/chain/chain/chain_makers.go | 2 +- test/chain/reward/main.go | 2 +- 75 files changed, 6343 insertions(+), 1251 deletions(-) create mode 100644 core/rawdb/accessors_snapshot.go create mode 100644 core/rawdb/accessors_state.go create mode 100644 core/rawdb/accessors_sync.go create mode 100644 core/rawdb/accessors_trie.go create mode 100644 core/rawdb/ancient_scheme.go create mode 100644 core/rawdb/ancient_utils.go create mode 100644 core/rawdb/chain_iterator.go create mode 100644 core/rawdb/database.go create mode 100644 core/rawdb/database_test.go create mode 100644 core/rawdb/databases_64bit.go create mode 100644 core/rawdb/databases_non64bit.go create mode 100644 core/rawdb/key_length_iterator.go create mode 100644 core/rawdb/key_length_iterator_test.go create mode 100644 core/rawdb/table.go create mode 100644 core/rawdb/table_test.go create mode 100644 core/rawdb/testdata/stored_receipts.bin create mode 100644 core/state/access_list.go create mode 100644 core/state/iterator.go create mode 100644 core/state/iterator_test.go create mode 100644 core/state/metrics.go create mode 100644 core/state/state_object_test.go create mode 100644 core/state/sync.go create mode 100644 core/state/transient_storage.go create mode 100644 core/state/trie_prefetcher.go create mode 100644 core/state/trie_prefetcher_test.go diff --git a/accounts/abi/bind/util_test.go b/accounts/abi/bind/util_test.go index 9f9b7a000d..75fbc91ceb 100644 --- a/accounts/abi/bind/util_test.go +++ b/accounts/abi/bind/util_test.go @@ -56,14 +56,17 @@ func TestWaitDeployed(t *testing.T) { for name, test := range waitDeployedTests { backend := backends.NewSimulatedBackend( core.GenesisAlloc{ - crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000)}, + crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000000000)}, }, 10000000, ) defer backend.Close() - // Create the transaction. - tx := types.NewContractCreation(0, big.NewInt(0), test.gas, big.NewInt(1), common.FromHex(test.code)) + // Create the transaction + head, _ := backend.HeaderByNumber(context.Background(), nil) // Should be child's, good enough + gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) + + tx := types.NewContractCreation(0, big.NewInt(0), test.gas, gasPrice, common.FromHex(test.code)) tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) // Wait for it to get mined in the background. @@ -99,15 +102,18 @@ func TestWaitDeployed(t *testing.T) { func TestWaitDeployedCornerCases(t *testing.T) { backend := backends.NewSimulatedBackend( core.GenesisAlloc{ - crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000)}, + crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000000000)}, }, 10000000, ) defer backend.Close() + head, _ := backend.HeaderByNumber(context.Background(), nil) // Should be child's, good enough + gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) + // Create a transaction to an account. code := "6060604052600a8060106000396000f360606040526008565b00" - tx := types.NewTransaction(0, common.HexToAddress("0x01"), big.NewInt(0), 3000000, big.NewInt(1), common.FromHex(code)) + tx := types.NewTransaction(0, common.HexToAddress("0x01"), big.NewInt(0), 3000000, gasPrice, common.FromHex(code)) tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -119,7 +125,7 @@ func TestWaitDeployedCornerCases(t *testing.T) { } // Create a transaction that is not mined. - tx = types.NewContractCreation(1, big.NewInt(0), 3000000, big.NewInt(1), common.FromHex(code)) + tx = types.NewContractCreation(1, big.NewInt(0), 3000000, gasPrice, common.FromHex(code)) tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) go func() { diff --git a/accounts/external/backend.go b/accounts/external/backend.go index dbfdd39d54..fb1fcffa35 100644 --- a/accounts/external/backend.go +++ b/accounts/external/backend.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/signer/core" + "github.com/ethereum/go-ethereum/signer/core/apitypes" "github.com/harmony-one/harmony/accounts" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/eth/rpc" @@ -218,12 +218,13 @@ func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transactio t := common.NewMixedcaseAddress(*tx.To()) to = &t } - args := &core.SendTxArgs{ + gas := hexutil.Big(*tx.GasPrice()) + args := &apitypes.SendTxArgs{ Data: &data, Nonce: hexutil.Uint64(tx.Nonce()), Value: hexutil.Big(*tx.Value()), Gas: hexutil.Uint64(tx.GasLimit()), - GasPrice: hexutil.Big(*tx.GasPrice()), + GasPrice: &gas, To: to, From: common.NewMixedcaseAddress(account.Address), } diff --git a/api/service/explorer/interface.go b/api/service/explorer/interface.go index a8e179415e..aa8a9ac856 100644 --- a/api/service/explorer/interface.go +++ b/api/service/explorer/interface.go @@ -42,7 +42,7 @@ type explorerDB struct { // newExplorerLvlDB new explorer storage using leveldb func newExplorerLvlDB(dbPath string) (database, error) { - db, err := leveldb.New(dbPath, 16, 500, "explorer_db") + db, err := leveldb.New(dbPath, 16, 500, "explorer_db", false) if err != nil { return nil, err } @@ -80,7 +80,7 @@ func (db *explorerDB) NewBatch() batch { } func (db *explorerDB) NewPrefixIterator(prefix []byte) iterator { - it := db.db.NewIteratorWithPrefix(prefix) + it := db.db.NewIterator(prefix, nil) return it } @@ -95,7 +95,7 @@ type sizedIterator struct { } func (db *explorerDB) newSizedIterator(start []byte, size int) *sizedIterator { - it := db.db.NewIteratorWithStart(start) + it := db.db.NewIterator(nil, start) return &sizedIterator{ it: it, curIndex: 0, diff --git a/cmd/harmony/dumpdb.go b/cmd/harmony/dumpdb.go index e66d3617bb..72e9de0300 100644 --- a/cmd/harmony/dumpdb.go +++ b/cmd/harmony/dumpdb.go @@ -176,7 +176,8 @@ func (db *KakashiDB) Close() error { return db.Database.Close() } -func (db *KakashiDB) OnRoot(common.Hash) {} +func (db *KakashiDB) OnRoot(common.Hash) {} +func (db *KakashiDB) OnAccount(common.Address, state.DumpAccount) {} // OnAccount implements DumpCollector interface func (db *KakashiDB) OnAccountStart(addr common.Address, acc state.DumpAccount) { @@ -345,7 +346,7 @@ func (db *KakashiDB) stateDataDump(block *types.Block) { fmt.Println("stateDataDump:", snapdbInfo.LastAccountKey.String(), snapdbInfo.LastAccountStateKey.String()) stateDB0 := state.NewDatabaseWithCache(db, STATEDB_CACHE_SIZE) rootHash := block.Root() - stateDB, err := state.New(rootHash, stateDB0) + stateDB, err := state.New(rootHash, stateDB0, nil) if err != nil { panic(err) } @@ -354,8 +355,8 @@ func (db *KakashiDB) stateDataDump(block *types.Block) { if len(snapdbInfo.LastAccountStateKey) > 0 { stateKey := new(big.Int).SetBytes(snapdbInfo.LastAccountStateKey) stateKey.Add(stateKey, big.NewInt(1)) - config.StateStart = stateKey.Bytes() - if len(config.StateStart) != len(snapdbInfo.LastAccountStateKey) { + config.Start = stateKey.Bytes() + if len(config.Start) != len(snapdbInfo.LastAccountStateKey) { panic("statekey overflow") } } @@ -366,12 +367,12 @@ func (db *KakashiDB) stateDataDump(block *types.Block) { func dumpMain(srcDBDir, destDBDir string, batchLimit int) { fmt.Println("===dumpMain===") - srcDB, err := ethRawDB.NewLevelDBDatabase(srcDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "") + srcDB, err := ethRawDB.NewLevelDBDatabase(srcDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "", false) if err != nil { fmt.Println("open src db error:", err) os.Exit(-1) } - destDB, err := ethRawDB.NewLevelDBDatabase(destDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "") + destDB, err := ethRawDB.NewLevelDBDatabase(destDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "", false) if err != nil { fmt.Println("open dest db error:", err) os.Exit(-1) diff --git a/contracts/Puzzle.go b/contracts/Puzzle.go index 916ae81950..de9a0334b2 100644 --- a/contracts/Puzzle.go +++ b/contracts/Puzzle.go @@ -20,7 +20,6 @@ var ( _ = big.NewInt _ = strings.NewReader _ = ethereum.NotFound - _ = abi.U256 _ = bind.Bind _ = common.Big1 _ = types.BloomLookup @@ -154,7 +153,7 @@ func bindPuzzle(address common.Address, caller bind.ContractCaller, transactor b // sets the output to result. The result type might be a single field for simple // returns, a slice of interfaces for anonymous returns and a struct for named // returns. -func (_Puzzle *PuzzleRaw) Call(opts *bind.CallOpts, result interface{}, method string, params ...interface{}) error { +func (_Puzzle *PuzzleRaw) Call(opts *bind.CallOpts, result *[]interface{}, method string, params ...interface{}) error { return _Puzzle.Contract.PuzzleCaller.contract.Call(opts, result, method, params...) } @@ -173,7 +172,7 @@ func (_Puzzle *PuzzleRaw) Transact(opts *bind.TransactOpts, method string, param // sets the output to result. The result type might be a single field for simple // returns, a slice of interfaces for anonymous returns and a struct for named // returns. -func (_Puzzle *PuzzleCallerRaw) Call(opts *bind.CallOpts, result interface{}, method string, params ...interface{}) error { +func (_Puzzle *PuzzleCallerRaw) Call(opts *bind.CallOpts, result *[]interface{}, method string, params ...interface{}) error { return _Puzzle.Contract.contract.Call(opts, result, method, params...) } @@ -192,12 +191,13 @@ func (_Puzzle *PuzzleTransactorRaw) Transact(opts *bind.TransactOpts, method str // // Solidity: function getPlayers() constant returns(address[]) func (_Puzzle *PuzzleCaller) GetPlayers(opts *bind.CallOpts) ([]common.Address, error) { - var ( - ret0 = new([]common.Address) - ) - out := ret0 - err := _Puzzle.contract.Call(opts, out, "getPlayers") - return *ret0, err + var results []interface{} + err := _Puzzle.contract.Call(opts, &results, "getPlayers") + if err != nil { + return *new([]common.Address), err + } + out := *abi.ConvertType(results, new([]common.Address)).(*[]common.Address) + return out, err } // GetPlayers is a free data retrieval call binding the contract method 0x8b5b9ccc. @@ -218,12 +218,13 @@ func (_Puzzle *PuzzleCallerSession) GetPlayers() ([]common.Address, error) { // // Solidity: function manager() constant returns(address) func (_Puzzle *PuzzleCaller) Manager(opts *bind.CallOpts) (common.Address, error) { - var ( - ret0 = new(common.Address) - ) - out := ret0 - err := _Puzzle.contract.Call(opts, out, "manager") - return *ret0, err + var results []interface{} + err := _Puzzle.contract.Call(opts, &results, "manager") + if err != nil { + return *new(common.Address), err + } + out := *abi.ConvertType(results[0], new([]common.Address)).(*[]common.Address) + return out[0], err } // Manager is a free data retrieval call binding the contract method 0x481c6a75. @@ -244,12 +245,13 @@ func (_Puzzle *PuzzleCallerSession) Manager() (common.Address, error) { // // Solidity: function players(uint256 ) constant returns(address) func (_Puzzle *PuzzleCaller) Players(opts *bind.CallOpts, arg0 *big.Int) (common.Address, error) { - var ( - ret0 = new(common.Address) - ) - out := ret0 - err := _Puzzle.contract.Call(opts, out, "players", arg0) - return *ret0, err + var results []interface{} + err := _Puzzle.contract.Call(opts, &results, "players", arg0) + if err != nil { + return *new(common.Address), err + } + out := *abi.ConvertType(results[0], new([]common.Address)).(*[]common.Address) + return out[0], err } // Players is a free data retrieval call binding the contract method 0xf71d96cb. diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 8b0683bd65..148176f236 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -135,9 +135,9 @@ type BlockChainImpl struct { pruneBeaconChainEnable bool // pruneBeaconChainEnable is enable prune BeaconChain feature shardID uint32 // Shard number - db ethdb.Database // Low level persistent database to store final content in - triegc *prque.Prque // Priority queue mapping block numbers to tries to gc - gcproc time.Duration // Accumulates canonical block processing for trie dumping + db ethdb.Database // Low level persistent database to store final content in + triegc *prque.Prque[int64, common.Hash] // Priority queue mapping block numbers to tries to gc + gcproc time.Duration // Accumulates canonical block processing for trie dumping // The following two variables are used to clean up the cache of redis in tikv mode. // This can improve the cache hit rate of redis @@ -246,7 +246,7 @@ func newBlockChainWithOptions( chainConfig: chainConfig, cacheConfig: cacheConfig, db: db, - triegc: prque.New(nil), + triegc: prque.New[int64, common.Hash](nil), stateCache: stateCache, quit: make(chan struct{}), bodyCache: bodyCache, @@ -462,7 +462,7 @@ func (bc *BlockChainImpl) ValidateNewBlock(block *types.Block, beaconChain Block } func (bc *BlockChainImpl) validateNewBlock(block *types.Block) error { - state, err := state.New(bc.CurrentBlock().Root(), bc.stateCache) + state, err := state.New(bc.CurrentBlock().Root(), bc.stateCache, nil) if err != nil { return err } @@ -520,7 +520,7 @@ func (bc *BlockChainImpl) loadLastState() error { return bc.Reset() } // Make sure the state associated with the block is available - if _, err := state.New(currentBlock.Root(), bc.stateCache); err != nil { + if _, err := state.New(currentBlock.Root(), bc.stateCache, nil); err != nil { // Dangling block without a state associated, init from scratch utils.Logger().Warn(). Str("number", currentBlock.Number().String()). @@ -617,7 +617,7 @@ func (bc *BlockChainImpl) SetHead(head uint64) error { headBlockGauge.Update(int64(newHeadBlock.NumberU64())) } if currentBlock := bc.CurrentBlock(); currentBlock != nil { - if _, err := state.New(currentBlock.Root(), bc.stateCache); err != nil { + if _, err := state.New(currentBlock.Root(), bc.stateCache, nil); err != nil { // Rewound state missing, rolled back to before pivot, reset to genesis bc.currentBlock.Store(bc.genesisBlock) headBlockGauge.Update(int64(bc.genesisBlock.NumberU64())) @@ -694,7 +694,7 @@ func (bc *BlockChainImpl) State() (*state.DB, error) { } func (bc *BlockChainImpl) StateAt(root common.Hash) (*state.DB, error) { - return state.New(root, bc.stateCache) + return state.New(root, bc.stateCache, nil) } func (bc *BlockChainImpl) Reset() error { @@ -739,7 +739,7 @@ func (bc *BlockChainImpl) repair(head **types.Block) error { valsToRemove := map[common.Address]struct{}{} for { // Abort if we've rewound to a head block that does have associated state - if _, err := state.New((*head).Root(), bc.stateCache); err == nil { + if _, err := state.New((*head).Root(), bc.stateCache, nil); err == nil { utils.Logger().Info(). Str("number", (*head).Number().String()). Str("hash", (*head).Hash().Hex()). @@ -1007,7 +1007,7 @@ func (bc *BlockChainImpl) GetReceiptsByHash(hash common.Hash) types.Receipts { return nil } - receipts := rawdb.ReadReceipts(bc.db, hash, *number) + receipts := rawdb.ReadReceipts(bc.db, hash, *number, nil) bc.receiptsCache.Add(hash, receipts) return receipts } @@ -1091,7 +1091,8 @@ func (bc *BlockChainImpl) Stop() { } } for !bc.triegc.Empty() { - triedb.Dereference(bc.triegc.PopItem().(common.Hash)) + v := common.Hash(bc.triegc.PopItem()) + triedb.Dereference(v) } if size, _ := triedb.Size(); size != 0 { utils.Logger().Error().Msg("Dangling trie nodes after full cleanup") @@ -1400,6 +1401,7 @@ func (bc *BlockChainImpl) WriteBlockWithState( } else { // Full but not archive node, do proper garbage collection triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive + // r := common.Hash(root) bc.triegc.Push(root, -int64(block.NumberU64())) if current := block.NumberU64(); current > bc.cacheConfig.TriesInMemory { @@ -1442,7 +1444,7 @@ func (bc *BlockChainImpl) WriteBlockWithState( if -number > bc.maxGarbCollectedBlkNum { bc.maxGarbCollectedBlkNum = -number } - triedb.Dereference(root.(common.Hash)) + triedb.Dereference(root) } } } @@ -1473,7 +1475,7 @@ func (bc *BlockChainImpl) WriteBlockWithState( if err := rawdb.WriteCxLookupEntries(batch, block); err != nil { return NonStatTy, err } - if err := rawdb.WritePreimages(batch, block.NumberU64(), state.Preimages()); err != nil { + if err := rawdb.WritePreimages(batch, state.Preimages()); err != nil { return NonStatTy, err } @@ -1677,7 +1679,7 @@ func (bc *BlockChainImpl) insertChain(chain types.Blocks, verifyHeaders bool) (i } else { parent = chain[i-1] } - state, err := state.New(parent.Root(), bc.stateCache) + state, err := state.New(parent.Root(), bc.stateCache, nil) if err != nil { return i, events, coalescedLogs, err } @@ -3362,14 +3364,14 @@ func (bc *BlockChainImpl) tikvCleanCache() { for i := bc.latestCleanCacheNum + 1; i <= to; i++ { // build previous block statedb fromBlock := bc.GetBlockByNumber(i) - fromTrie, err := state.New(fromBlock.Root(), bc.stateCache) + fromTrie, err := state.New(fromBlock.Root(), bc.stateCache, nil) if err != nil { continue } // build current block statedb toBlock := bc.GetBlockByNumber(i + 1) - toTrie, err := state.New(toBlock.Root(), bc.stateCache) + toTrie, err := state.New(toBlock.Root(), bc.stateCache, nil) if err != nil { continue } @@ -3469,7 +3471,7 @@ func (bc *BlockChainImpl) InitTiKV(conf *harmonyconfig.TiKVConfig) { // If redis is empty, the hit rate will be too low and the synchronization block speed will be slow // set LOAD_PRE_FETCH is yes can significantly improve this. if os.Getenv("LOAD_PRE_FETCH") == "yes" { - if trie, err := state.New(bc.CurrentBlock().Root(), bc.stateCache); err == nil { + if trie, err := state.New(bc.CurrentBlock().Root(), bc.stateCache, nil); err == nil { trie.Prefetch(512) } else { log.Println("LOAD_PRE_FETCH ERR: ", err) diff --git a/core/blockchain_pruner.go b/core/blockchain_pruner.go index 0dbe6fb696..744a7d7061 100644 --- a/core/blockchain_pruner.go +++ b/core/blockchain_pruner.go @@ -43,6 +43,11 @@ func newBlockchainPruner(db ethdb.Database) *blockchainPruner { } } +// Put inserts the given value into the key-value data store. +func (bp *blockchainPruner) Put(key []byte, value []byte) error { + return nil +} + func (bp *blockchainPruner) Delete(key []byte) error { err := bp.batchWriter.Delete(key) if err != nil { diff --git a/core/genesis.go b/core/genesis.go index dec247a766..3bd4f718aa 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -244,7 +244,7 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { utils.Logger().Error().Msg("db should be initialized") os.Exit(1) } - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(db), nil) for addr, account := range g.Alloc { statedb.AddBalance(addr, account.Balance) statedb.SetCode(addr, account.Code) diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 106a6ff0a6..72ce358e29 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -19,13 +19,20 @@ package rawdb import ( "bytes" "encoding/binary" + "errors" + "fmt" "math/big" + "sort" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/block" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/utils" + + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" ) // MsgNoShardStateFromDB error message for shard state reading failure @@ -39,7 +46,7 @@ const ( ) // ReadCanonicalHash retrieves the hash assigned to a canonical block number. -func ReadCanonicalHash(db DatabaseReader, number uint64) common.Hash { +func ReadCanonicalHash(db ethdb.Reader, number uint64) common.Hash { data, _ := db.Get(headerHashKey(number)) if len(data) == 0 { return common.Hash{} @@ -48,7 +55,7 @@ func ReadCanonicalHash(db DatabaseReader, number uint64) common.Hash { } // WriteCanonicalHash stores the hash assigned to a canonical block number. -func WriteCanonicalHash(db DatabaseWriter, hash common.Hash, number uint64) error { +func WriteCanonicalHash(db ethdb.KeyValueWriter, hash common.Hash, number uint64) error { if err := db.Put(headerHashKey(number), hash.Bytes()); err != nil { utils.Logger().Error().Msg("Failed to store number to hash mapping") return err @@ -57,7 +64,7 @@ func WriteCanonicalHash(db DatabaseWriter, hash common.Hash, number uint64) erro } // DeleteCanonicalHash removes the number to hash canonical mapping. -func DeleteCanonicalHash(db DatabaseDeleter, number uint64) error { +func DeleteCanonicalHash(db ethdb.KeyValueWriter, number uint64) error { if err := db.Delete(headerHashKey(number)); err != nil { utils.Logger().Error().Msg("Failed to delete number to hash mapping") return err @@ -66,7 +73,7 @@ func DeleteCanonicalHash(db DatabaseDeleter, number uint64) error { } // ReadHeaderNumber returns the header number assigned to a hash. -func ReadHeaderNumber(db DatabaseReader, hash common.Hash) *uint64 { +func ReadHeaderNumber(db ethdb.KeyValueReader, hash common.Hash) *uint64 { data, _ := db.Get(headerNumberKey(hash)) if len(data) != 8 { return nil @@ -76,7 +83,7 @@ func ReadHeaderNumber(db DatabaseReader, hash common.Hash) *uint64 { } // WriteHeaderNumber stores reference from hash to number. -func WriteHeaderNumber(db DatabaseWriter, hash common.Hash, number uint64) error { +func WriteHeaderNumber(db ethdb.KeyValueWriter, hash common.Hash, number uint64) error { var ( key = headerNumberKey(hash) encoded = encodeBlockNumber(number) @@ -86,7 +93,7 @@ func WriteHeaderNumber(db DatabaseWriter, hash common.Hash, number uint64) error } // ReadHeadHeaderHash retrieves the hash of the current canonical head header. -func ReadHeadHeaderHash(db DatabaseReader) common.Hash { +func ReadHeadHeaderHash(db ethdb.KeyValueReader) common.Hash { data, _ := db.Get(headHeaderKey) if len(data) == 0 { return common.Hash{} @@ -95,7 +102,7 @@ func ReadHeadHeaderHash(db DatabaseReader) common.Hash { } // WriteHeadHeaderHash stores the hash of the current canonical head header. -func WriteHeadHeaderHash(db DatabaseWriter, hash common.Hash) error { +func WriteHeadHeaderHash(db ethdb.KeyValueWriter, hash common.Hash) error { if err := db.Put(headHeaderKey, hash.Bytes()); err != nil { utils.Logger().Error().Msg("Failed to store last header's hash") return err @@ -104,7 +111,7 @@ func WriteHeadHeaderHash(db DatabaseWriter, hash common.Hash) error { } // ReadHeadBlockHash retrieves the hash of the current canonical head block. -func ReadHeadBlockHash(db DatabaseReader) common.Hash { +func ReadHeadBlockHash(db ethdb.KeyValueReader) common.Hash { data, _ := db.Get(headBlockKey) if len(data) == 0 { return common.Hash{} @@ -113,7 +120,7 @@ func ReadHeadBlockHash(db DatabaseReader) common.Hash { } // WriteHeadBlockHash stores the head block's hash. -func WriteHeadBlockHash(db DatabaseWriter, hash common.Hash) error { +func WriteHeadBlockHash(db ethdb.KeyValueWriter, hash common.Hash) error { if err := db.Put(headBlockKey, hash.Bytes()); err != nil { utils.Logger().Error().Msg("Failed to store last block's hash") return err @@ -122,7 +129,7 @@ func WriteHeadBlockHash(db DatabaseWriter, hash common.Hash) error { } // ReadHeadFastBlockHash retrieves the hash of the current fast-sync head block. -func ReadHeadFastBlockHash(db DatabaseReader) common.Hash { +func ReadHeadFastBlockHash(db ethdb.KeyValueReader) common.Hash { data, _ := db.Get(headFastBlockKey) if len(data) == 0 { return common.Hash{} @@ -131,7 +138,7 @@ func ReadHeadFastBlockHash(db DatabaseReader) common.Hash { } // WriteHeadFastBlockHash stores the hash of the current fast-sync head block. -func WriteHeadFastBlockHash(db DatabaseWriter, hash common.Hash) error { +func WriteHeadFastBlockHash(db ethdb.KeyValueWriter, hash common.Hash) error { if err := db.Put(headFastBlockKey, hash.Bytes()); err != nil { utils.Logger().Error().Msg("Failed to store last fast block's hash") return err @@ -140,13 +147,13 @@ func WriteHeadFastBlockHash(db DatabaseWriter, hash common.Hash) error { } // ReadHeaderRLP retrieves a block header in its raw RLP database encoding. -func ReadHeaderRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { +func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { data, _ := db.Get(headerKey(number, hash)) return data } // HasHeader verifies the existence of a block header corresponding to the hash. -func HasHeader(db DatabaseReader, hash common.Hash, number uint64) bool { +func HasHeader(db ethdb.Reader, hash common.Hash, number uint64) bool { if has, err := db.Has(headerKey(number, hash)); !has || err != nil { return false } @@ -154,7 +161,7 @@ func HasHeader(db DatabaseReader, hash common.Hash, number uint64) bool { } // ReadHeader retrieves the block header corresponding to the hash. -func ReadHeader(db DatabaseReader, hash common.Hash, number uint64) *block.Header { +func ReadHeader(db ethdb.Reader, hash common.Hash, number uint64) *block.Header { data := ReadHeaderRLP(db, hash, number) if len(data) == 0 { return nil @@ -169,7 +176,7 @@ func ReadHeader(db DatabaseReader, hash common.Hash, number uint64) *block.Heade // WriteHeader stores a block header into the database and also stores the hash- // to-number mapping. -func WriteHeader(db DatabaseWriter, header *block.Header) error { +func WriteHeader(db ethdb.KeyValueWriter, header *block.Header) error { // Write the hash -> number mapping var ( hash = header.Hash() @@ -196,7 +203,7 @@ func WriteHeader(db DatabaseWriter, header *block.Header) error { } // DeleteHeader removes all block header data associated with a hash. -func DeleteHeader(db DatabaseDeleter, hash common.Hash, number uint64) error { +func DeleteHeader(db ethdb.KeyValueWriter, hash common.Hash, number uint64) error { if err := db.Delete(headerKey(number, hash)); err != nil { utils.Logger().Error().Msg("Failed to delete header") return err @@ -209,13 +216,13 @@ func DeleteHeader(db DatabaseDeleter, hash common.Hash, number uint64) error { } // ReadBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. -func ReadBodyRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { +func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { data, _ := db.Get(blockBodyKey(number, hash)) return data } // WriteBodyRLP stores an RLP encoded block body into the database. -func WriteBodyRLP(db DatabaseWriter, hash common.Hash, number uint64, rlp rlp.RawValue) error { +func WriteBodyRLP(db ethdb.KeyValueWriter, hash common.Hash, number uint64, rlp rlp.RawValue) error { if err := db.Put(blockBodyKey(number, hash), rlp); err != nil { utils.Logger().Error().Msg("Failed to store block body") return err @@ -224,7 +231,7 @@ func WriteBodyRLP(db DatabaseWriter, hash common.Hash, number uint64, rlp rlp.Ra } // HasBody verifies the existence of a block body corresponding to the hash. -func HasBody(db DatabaseReader, hash common.Hash, number uint64) bool { +func HasBody(db ethdb.Reader, hash common.Hash, number uint64) bool { if has, err := db.Has(blockBodyKey(number, hash)); !has || err != nil { return false } @@ -232,7 +239,7 @@ func HasBody(db DatabaseReader, hash common.Hash, number uint64) bool { } // ReadBody retrieves the block body corresponding to the hash. -func ReadBody(db DatabaseReader, hash common.Hash, number uint64) *types.Body { +func ReadBody(db ethdb.Reader, hash common.Hash, number uint64) *types.Body { data := ReadBodyRLP(db, hash, number) if len(data) == 0 { return nil @@ -246,7 +253,7 @@ func ReadBody(db DatabaseReader, hash common.Hash, number uint64) *types.Body { } // WriteBody storea a block body into the database. -func WriteBody(db DatabaseWriter, hash common.Hash, number uint64, body *types.Body) error { +func WriteBody(db ethdb.KeyValueWriter, hash common.Hash, number uint64, body *types.Body) error { data, err := rlp.EncodeToBytes(body) if err != nil { utils.Logger().Error().Msg("Failed to RLP encode body") @@ -256,7 +263,7 @@ func WriteBody(db DatabaseWriter, hash common.Hash, number uint64, body *types.B } // DeleteBody removes all block body data associated with a hash. -func DeleteBody(db DatabaseDeleter, hash common.Hash, number uint64) error { +func DeleteBody(db ethdb.KeyValueWriter, hash common.Hash, number uint64) error { if err := db.Delete(blockBodyKey(number, hash)); err != nil { utils.Logger().Error().Msg("Failed to delete block body") return err @@ -265,7 +272,7 @@ func DeleteBody(db DatabaseDeleter, hash common.Hash, number uint64) error { } // ReadTd retrieves a block's total difficulty corresponding to the hash. -func ReadTd(db DatabaseReader, hash common.Hash, number uint64) *big.Int { +func ReadTd(db ethdb.Reader, hash common.Hash, number uint64) *big.Int { data, _ := db.Get(headerTDKey(number, hash)) if len(data) == 0 { return nil @@ -279,7 +286,7 @@ func ReadTd(db DatabaseReader, hash common.Hash, number uint64) *big.Int { } // WriteTd stores the total difficulty of a block into the database. -func WriteTd(db DatabaseWriter, hash common.Hash, number uint64, td *big.Int) error { +func WriteTd(db ethdb.KeyValueWriter, hash common.Hash, number uint64, td *big.Int) error { data, err := rlp.EncodeToBytes(td) if err != nil { utils.Logger().Error().Msg("Failed to RLP encode block total difficulty") @@ -293,7 +300,7 @@ func WriteTd(db DatabaseWriter, hash common.Hash, number uint64, td *big.Int) er } // DeleteTd removes all block total difficulty data associated with a hash. -func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) error { +func DeleteTd(db ethdb.KeyValueWriter, hash common.Hash, number uint64) error { if err := db.Delete(headerTDKey(number, hash)); err != nil { utils.Logger().Error().Msg("Failed to delete block total difficulty") return err @@ -302,7 +309,7 @@ func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) error { } // ReadReceipts retrieves all the transaction receipts belonging to a block. -func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts { +func ReadReceipts(db ethdb.Reader, hash common.Hash, number uint64, config *params.ChainConfig) types.Receipts { // Retrieve the flattened receipt slice data, _ := db.Get(blockReceiptsKey(number, hash)) if len(data) == 0 { @@ -321,8 +328,24 @@ func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Rece return receipts } +// ReadReceiptsRLP retrieves all the transaction receipts belonging to a block in RLP encoding. +func ReadReceiptsRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + var data []byte + db.ReadAncients(func(reader ethdb.AncientReaderOp) error { + // Check if the data is in ancients + if isCanon(reader, number, hash) { + data, _ = reader.Ancient(ChainFreezerReceiptTable, number) + return nil + } + // If not, try reading from leveldb + data, _ = db.Get(blockReceiptsKey(number, hash)) + return nil + }) + return data +} + // WriteReceipts stores all the transaction receipts belonging to a block. -func WriteReceipts(db DatabaseWriter, hash common.Hash, number uint64, receipts types.Receipts) error { +func WriteReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64, receipts types.Receipts) error { // Convert the receipts into their storage form and serialize them storageReceipts := make([]*types.ReceiptForStorage, len(receipts)) for i, receipt := range receipts { @@ -342,7 +365,7 @@ func WriteReceipts(db DatabaseWriter, hash common.Hash, number uint64, receipts } // DeleteReceipts removes all receipt data associated with a block hash. -func DeleteReceipts(db DatabaseDeleter, hash common.Hash, number uint64) error { +func DeleteReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64) error { if err := db.Delete(blockReceiptsKey(number, hash)); err != nil { utils.Logger().Error().Msg("Failed to delete block receipts") return err @@ -356,7 +379,7 @@ func DeleteReceipts(db DatabaseDeleter, hash common.Hash, number uint64) error { // // Note, due to concurrent download of header and block body the header and thus // canonical hash can be stored in the database but the body data not (yet). -func ReadBlock(db DatabaseReader, hash common.Hash, number uint64) *types.Block { +func ReadBlock(db ethdb.Reader, hash common.Hash, number uint64) *types.Block { header := ReadHeader(db, hash, number) if header == nil { return nil @@ -369,7 +392,7 @@ func ReadBlock(db DatabaseReader, hash common.Hash, number uint64) *types.Block } // WriteBlock serializes a block into the database, header and body separately. -func WriteBlock(db DatabaseWriter, block *types.Block) error { +func WriteBlock(db ethdb.KeyValueWriter, block *types.Block) error { if err := WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { return err } @@ -387,7 +410,7 @@ func WriteBlock(db DatabaseWriter, block *types.Block) error { } // DeleteBlock removes all block data associated with a hash. -func DeleteBlock(db DatabaseDeleter, hash common.Hash, number uint64) error { +func DeleteBlock(db ethdb.KeyValueWriter, hash common.Hash, number uint64) error { if err := DeleteReceipts(db, hash, number); err != nil { return err } @@ -404,7 +427,7 @@ func DeleteBlock(db DatabaseDeleter, hash common.Hash, number uint64) error { } // FindCommonAncestor returns the last common ancestor of two block headers -func FindCommonAncestor(db DatabaseReader, a, b *block.Header) *block.Header { +func FindCommonAncestor(db ethdb.Reader, a, b *block.Header) *block.Header { for bn := b.Number().Uint64(); a.Number().Uint64() > bn; { a = ReadHeader(db, a.ParentHash(), a.Number().Uint64()-1) if a == nil { @@ -431,7 +454,7 @@ func FindCommonAncestor(db DatabaseReader, a, b *block.Header) *block.Header { } func IteratorBlocks(iterator DatabaseIterator, cb func(blockNum uint64, hash common.Hash) bool) (minKey []byte, maxKey []byte) { - iter := iterator.NewIteratorWithPrefix(headerPrefix) + iter := iterator.NewIterator(headerPrefix, nil) defer iter.Release() minKey = headerPrefix @@ -453,3 +476,443 @@ func IteratorBlocks(iterator DatabaseIterator, cb func(blockNum uint64, hash com return } + +// ReadAllHashes retrieves all the hashes assigned to blocks at a certain heights, +// both canonical and reorged forks included. +func ReadAllHashes(db ethdb.Iteratee, number uint64) []common.Hash { + prefix := headerKeyPrefix(number) + + hashes := make([]common.Hash, 0, 1) + it := db.NewIterator(prefix, nil) + defer it.Release() + + for it.Next() { + if key := it.Key(); len(key) == len(prefix)+32 { + hashes = append(hashes, common.BytesToHash(key[len(key)-32:])) + } + } + return hashes +} + +type NumberHash struct { + Number uint64 + Hash common.Hash +} + +// ReadAllHashesInRange retrieves all the hashes assigned to blocks at certain +// heights, both canonical and reorged forks included. +// This method considers both limits to be _inclusive_. +func ReadAllHashesInRange(db ethdb.Iteratee, first, last uint64) []*NumberHash { + var ( + start = encodeBlockNumber(first) + keyLength = len(headerPrefix) + 8 + 32 + hashes = make([]*NumberHash, 0, 1+last-first) + it = db.NewIterator(headerPrefix, start) + ) + defer it.Release() + for it.Next() { + key := it.Key() + if len(key) != keyLength { + continue + } + num := binary.BigEndian.Uint64(key[len(headerPrefix) : len(headerPrefix)+8]) + if num > last { + break + } + hash := common.BytesToHash(key[len(key)-32:]) + hashes = append(hashes, &NumberHash{num, hash}) + } + return hashes +} + +// ReadAllCanonicalHashes retrieves all canonical number and hash mappings at the +// certain chain range. If the accumulated entries reaches the given threshold, +// abort the iteration and return the semi-finish result. +func ReadAllCanonicalHashes(db ethdb.Iteratee, from uint64, to uint64, limit int) ([]uint64, []common.Hash) { + // Short circuit if the limit is 0. + if limit == 0 { + return nil, nil + } + var ( + numbers []uint64 + hashes []common.Hash + ) + // Construct the key prefix of start point. + start, end := headerHashKey(from), headerHashKey(to) + it := db.NewIterator(nil, start) + defer it.Release() + + for it.Next() { + if bytes.Compare(it.Key(), end) >= 0 { + break + } + if key := it.Key(); len(key) == len(headerPrefix)+8+1 && bytes.Equal(key[len(key)-1:], headerHashSuffix) { + numbers = append(numbers, binary.BigEndian.Uint64(key[len(headerPrefix):len(headerPrefix)+8])) + hashes = append(hashes, common.BytesToHash(it.Value())) + // If the accumulated entries reaches the limit threshold, return. + if len(numbers) >= limit { + break + } + } + } + return numbers, hashes +} + +// DeleteHeaderNumber removes hash->number mapping. +func DeleteHeaderNumber(db ethdb.KeyValueWriter, hash common.Hash) { + if err := db.Delete(headerNumberKey(hash)); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to delete hash to number mapping") + } +} + +// ReadFinalizedBlockHash retrieves the hash of the finalized block. +func ReadFinalizedBlockHash(db ethdb.KeyValueReader) common.Hash { + data, _ := db.Get(headFinalizedBlockKey) + if len(data) == 0 { + return common.Hash{} + } + return common.BytesToHash(data) +} + +// WriteFinalizedBlockHash stores the hash of the finalized block. +func WriteFinalizedBlockHash(db ethdb.KeyValueWriter, hash common.Hash) { + if err := db.Put(headFinalizedBlockKey, hash.Bytes()); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store last finalized block's hash") + } +} + +// ReadLastPivotNumber retrieves the number of the last pivot block. If the node +// full synced, the last pivot will always be nil. +func ReadLastPivotNumber(db ethdb.KeyValueReader) *uint64 { + data, _ := db.Get(lastPivotKey) + if len(data) == 0 { + return nil + } + var pivot uint64 + if err := rlp.DecodeBytes(data, &pivot); err != nil { + utils.Logger().Error().Err(err).Msg("Invalid pivot block number in database") + return nil + } + return &pivot +} + +// WriteLastPivotNumber stores the number of the last pivot block. +func WriteLastPivotNumber(db ethdb.KeyValueWriter, pivot uint64) { + enc, err := rlp.EncodeToBytes(pivot) + if err != nil { + utils.Logger().Error().Err(err).Msg("Failed to encode pivot block number") + } + if err := db.Put(lastPivotKey, enc); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store pivot block number") + } +} + +// ReadTxIndexTail retrieves the number of oldest indexed block +// whose transaction indices has been indexed. +func ReadTxIndexTail(db ethdb.KeyValueReader) *uint64 { + data, _ := db.Get(txIndexTailKey) + if len(data) != 8 { + return nil + } + number := binary.BigEndian.Uint64(data) + return &number +} + +// WriteTxIndexTail stores the number of oldest indexed block +// into database. +func WriteTxIndexTail(db ethdb.KeyValueWriter, number uint64) { + if err := db.Put(txIndexTailKey, encodeBlockNumber(number)); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store the transaction index tail") + } +} + +// ReadFastTxLookupLimit retrieves the tx lookup limit used in fast sync. +func ReadFastTxLookupLimit(db ethdb.KeyValueReader) *uint64 { + data, _ := db.Get(fastTxLookupLimitKey) + if len(data) != 8 { + return nil + } + number := binary.BigEndian.Uint64(data) + return &number +} + +// WriteFastTxLookupLimit stores the txlookup limit used in fast sync into database. +func WriteFastTxLookupLimit(db ethdb.KeyValueWriter, number uint64) { + if err := db.Put(fastTxLookupLimitKey, encodeBlockNumber(number)); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store transaction lookup limit for fast sync") + } +} + +// deleteHeaderWithoutNumber removes only the block header but does not remove +// the hash to number mapping. +func deleteHeaderWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { + if err := db.Delete(headerKey(number, hash)); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to delete header") + } +} + +// isCanon is an internal utility method, to check whether the given number/hash +// is part of the ancient (canon) set. +func isCanon(reader ethdb.AncientReaderOp, number uint64, hash common.Hash) bool { + h, err := reader.Ancient(ChainFreezerHashTable, number) + if err != nil { + return false + } + return bytes.Equal(h, hash[:]) +} + +// ReadCanonicalBodyRLP retrieves the block body (transactions and uncles) for the canonical +// block at number, in RLP encoding. +func ReadCanonicalBodyRLP(db ethdb.Reader, number uint64) rlp.RawValue { + var data []byte + db.ReadAncients(func(reader ethdb.AncientReaderOp) error { + data, _ = reader.Ancient(ChainFreezerBodiesTable, number) + if len(data) > 0 { + return nil + } + // Block is not in ancients, read from leveldb by hash and number. + // Note: ReadCanonicalHash cannot be used here because it also + // calls ReadAncients internally. + hash, _ := db.Get(headerHashKey(number)) + data, _ = db.Get(blockBodyKey(number, common.BytesToHash(hash))) + return nil + }) + return data +} + +// ReadTdRLP retrieves a block's total difficulty corresponding to the hash in RLP encoding. +func ReadTdRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + var data []byte + db.ReadAncients(func(reader ethdb.AncientReaderOp) error { + // Check if the data is in ancients + if isCanon(reader, number, hash) { + data, _ = reader.Ancient(ChainFreezerDifficultyTable, number) + return nil + } + // If not, try reading from leveldb + data, _ = db.Get(headerTDKey(number, hash)) + return nil + }) + return data +} + +// HasReceipts verifies the existence of all the transaction receipts belonging +// to a block. +func HasReceipts(db ethdb.Reader, hash common.Hash, number uint64) bool { + if isCanon(db, number, hash) { + return true + } + if has, err := db.Has(blockReceiptsKey(number, hash)); !has || err != nil { + return false + } + return true +} + +// ReadRawReceipts retrieves all the transaction receipts belonging to a block. +// The receipt metadata fields are not guaranteed to be populated, so they +// should not be used. Use ReadReceipts instead if the metadata is needed. +func ReadRawReceipts(db ethdb.Reader, hash common.Hash, number uint64) types.Receipts { + // Retrieve the flattened receipt slice + data := ReadReceiptsRLP(db, hash, number) + if len(data) == 0 { + return nil + } + // Convert the receipts from their storage form to their internal representation + storageReceipts := []*types.ReceiptForStorage{} + if err := rlp.DecodeBytes(data, &storageReceipts); err != nil { + utils.Logger().Error().Err(err).Interface("hash", hash).Msg("Invalid receipt array RLP") + return nil + } + receipts := make(types.Receipts, len(storageReceipts)) + for i, storageReceipt := range storageReceipts { + receipts[i] = (*types.Receipt)(storageReceipt) + } + return receipts +} + +// storedReceiptRLP is the storage encoding of a receipt. +// Re-definition in core/types/receipt.go. +// TODO: Re-use the existing definition. +type storedReceiptRLP struct { + PostStateOrStatus []byte + CumulativeGasUsed uint64 + Logs []*types.Log +} + +// ReceiptLogs is a barebone version of ReceiptForStorage which only keeps +// the list of logs. When decoding a stored receipt into this object we +// avoid creating the bloom filter. +type receiptLogs struct { + Logs []*types.Log +} + +// DecodeRLP implements rlp.Decoder. +func (r *receiptLogs) DecodeRLP(s *rlp.Stream) error { + var stored storedReceiptRLP + if err := s.Decode(&stored); err != nil { + return err + } + r.Logs = stored.Logs + return nil +} + +// DeriveLogFields fills the logs in receiptLogs with information such as block number, txhash, etc. +func deriveLogFields(receipts []*receiptLogs, hash common.Hash, number uint64, txs types.Transactions) error { + logIndex := uint(0) + if len(txs) != len(receipts) { + return errors.New("transaction and receipt count mismatch") + } + for i := 0; i < len(receipts); i++ { + txHash := txs[i].Hash() + // The derived log fields can simply be set from the block and transaction + for j := 0; j < len(receipts[i].Logs); j++ { + receipts[i].Logs[j].BlockNumber = number + receipts[i].Logs[j].BlockHash = hash + receipts[i].Logs[j].TxHash = txHash + receipts[i].Logs[j].TxIndex = uint(i) + receipts[i].Logs[j].Index = logIndex + logIndex++ + } + } + return nil +} + +// ReadLogs retrieves the logs for all transactions in a block. In case +// receipts is not found, a nil is returned. +// Note: ReadLogs does not derive unstored log fields. +func ReadLogs(db ethdb.Reader, hash common.Hash, number uint64, config *params.ChainConfig) [][]*types.Log { + // Retrieve the flattened receipt slice + data := ReadReceiptsRLP(db, hash, number) + if len(data) == 0 { + return nil + } + receipts := []*receiptLogs{} + if err := rlp.DecodeBytes(data, &receipts); err != nil { + utils.Logger().Error().Err(err).Interface("hash", hash).Msg("Invalid receipt array RLP") + return nil + } + + logs := make([][]*types.Log, len(receipts)) + for i, receipt := range receipts { + logs[i] = receipt.Logs + } + return logs +} + +// This function is NOT used, just ported over from the Ethereum +func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *block.Header, receipts []*types.ReceiptForStorage, td *big.Int) error { + num := block.NumberU64() + if err := op.AppendRaw(ChainFreezerHashTable, num, block.Hash().Bytes()); err != nil { + return fmt.Errorf("can't add block %d hash: %v", num, err) + } + if err := op.Append(ChainFreezerHeaderTable, num, header); err != nil { + return fmt.Errorf("can't append block header %d: %v", num, err) + } + if err := op.Append(ChainFreezerBodiesTable, num, block.Body()); err != nil { + return fmt.Errorf("can't append block body %d: %v", num, err) + } + if err := op.Append(ChainFreezerReceiptTable, num, receipts); err != nil { + return fmt.Errorf("can't append block %d receipts: %v", num, err) + } + if err := op.Append(ChainFreezerDifficultyTable, num, td); err != nil { + return fmt.Errorf("can't append block %d total difficulty: %v", num, err) + } + return nil +} + +// DeleteBlockWithoutNumber removes all block data associated with a hash, except +// the hash to number mapping. +func DeleteBlockWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { + DeleteReceipts(db, hash, number) + deleteHeaderWithoutNumber(db, hash, number) + DeleteBody(db, hash, number) + DeleteTd(db, hash, number) +} + +const badBlockToKeep = 10 + +type badBlock struct { + Header *block.Header + Body *types.Body +} + +// badBlockList implements the sort interface to allow sorting a list of +// bad blocks by their number in the reverse order. +type badBlockList []*badBlock + +func (s badBlockList) Len() int { return len(s) } +func (s badBlockList) Less(i, j int) bool { + return s[i].Header.Number().Uint64() < s[j].Header.Number().Uint64() +} +func (s badBlockList) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// WriteBadBlock serializes the bad block into the database. If the cumulated +// bad blocks exceeds the limitation, the oldest will be dropped. +// This function is NOT used, just ported over from the Ethereum +func WriteBadBlock(db ethdb.KeyValueStore, block *types.Block) { + blob, err := db.Get(badBlockKey) + if err != nil { + utils.Logger().Warn().Err(err).Msg("Failed to load old bad blocks") + } + var badBlocks badBlockList + if len(blob) > 0 { + if err := rlp.DecodeBytes(blob, &badBlocks); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to decode old bad blocks") + } + } + for _, b := range badBlocks { + if b.Header.Number().Uint64() == block.NumberU64() && b.Header.Hash() == block.Hash() { + log.Info("Skip duplicated bad block", "number", block.NumberU64(), "hash", block.Hash()) + return + } + } + badBlocks = append(badBlocks, &badBlock{ + Header: block.Header(), + Body: block.Body(), + }) + sort.Sort(sort.Reverse(badBlocks)) + if len(badBlocks) > badBlockToKeep { + badBlocks = badBlocks[:badBlockToKeep] + } + data, err := rlp.EncodeToBytes(badBlocks) + if err != nil { + utils.Logger().Error().Err(err).Msg("Failed to encode bad blocks") + } + if err := db.Put(badBlockKey, data); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to write bad blocks") + } +} + +// DeleteBadBlocks deletes all the bad blocks from the database +// This function is NOT used, just ported over from the Ethereum +func DeleteBadBlocks(db ethdb.KeyValueWriter) { + if err := db.Delete(badBlockKey); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to delete bad blocks") + } +} + +// ReadHeadHeader returns the current canonical head header. +func ReadHeadHeader(db ethdb.Reader) *block.Header { + headHeaderHash := ReadHeadHeaderHash(db) + if headHeaderHash == (common.Hash{}) { + return nil + } + headHeaderNumber := ReadHeaderNumber(db, headHeaderHash) + if headHeaderNumber == nil { + return nil + } + return ReadHeader(db, headHeaderHash, *headHeaderNumber) +} + +// ReadHeadBlock returns the current canonical head block. +func ReadHeadBlock(db ethdb.Reader) *types.Block { + headBlockHash := ReadHeadBlockHash(db) + if headBlockHash == (common.Hash{}) { + return nil + } + headBlockNumber := ReadHeaderNumber(db, headBlockHash) + if headBlockNumber == nil { + return nil + } + return ReadBlock(db, headBlockHash, *headBlockNumber) +} diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index 9824367d21..e9dea08824 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -309,14 +309,14 @@ func TestBlockReceiptStorage(t *testing.T) { // Check that no receipt entries are in a pristine database hash := common.BytesToHash([]byte{0x03, 0x14}) - if rs := ReadReceipts(db, hash, 0); len(rs) != 0 { + if rs := ReadReceipts(db, hash, 0, nil); len(rs) != 0 { t.Fatalf("non existent receipts returned: %v", rs) } // Insert the receipt slice into the database and check presence if err := WriteReceipts(db, hash, 0, receipts); err != nil { t.Fatalf("write receipts") } - if rs := ReadReceipts(db, hash, 0); len(rs) == 0 { + if rs := ReadReceipts(db, hash, 0, nil); len(rs) == 0 { t.Fatalf("no receipts returned") } else { for i := 0; i < len(receipts); i++ { @@ -330,7 +330,7 @@ func TestBlockReceiptStorage(t *testing.T) { } // Delete the receipt slice and check purge DeleteReceipts(db, hash, 0) - if rs := ReadReceipts(db, hash, 0); len(rs) != 0 { + if rs := ReadReceipts(db, hash, 0, nil); len(rs) != 0 { t.Fatalf("deleted receipts returned: %v", rs) } } diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 44b2961ac9..2fcb61deb7 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -18,6 +18,7 @@ package rawdb import ( "bytes" + "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" @@ -25,11 +26,14 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/utils" staking "github.com/harmony-one/harmony/staking/types" + + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/params" ) // ReadTxLookupEntry retrieves the positional metadata associated with a transaction // hash to allow retrieving the transaction or receipt by hash. -func ReadTxLookupEntry(db DatabaseReader, hash common.Hash) (common.Hash, uint64, uint64) { +func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) (common.Hash, uint64, uint64) { data, _ := db.Get(txLookupKey(hash)) if len(data) == 0 { return common.Hash{}, 0, 0 @@ -88,13 +92,13 @@ func WriteBlockStxLookUpEntries(db DatabaseWriter, block *types.Block) error { } // DeleteTxLookupEntry removes all transaction data associated with a hash. -func DeleteTxLookupEntry(db DatabaseDeleter, hash common.Hash) error { +func DeleteTxLookupEntry(db ethdb.KeyValueWriter, hash common.Hash) error { return db.Delete(txLookupKey(hash)) } // ReadTransaction retrieves a specific transaction from the database, along with // its added positional metadata. -func ReadTransaction(db DatabaseReader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) { +func ReadTransaction(db ethdb.Reader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) { blockHash, blockNumber, txIndex := ReadTxLookupEntry(db, hash) if blockHash == (common.Hash{}) { return nil, common.Hash{}, 0, 0 @@ -135,7 +139,7 @@ func ReadTransaction(db DatabaseReader, hash common.Hash) (*types.Transaction, c // ReadStakingTransaction retrieves a specific staking transaction from the database, along with // its added positional metadata. -func ReadStakingTransaction(db DatabaseReader, hash common.Hash) (*staking.StakingTransaction, common.Hash, uint64, uint64) { +func ReadStakingTransaction(db ethdb.Reader, hash common.Hash) (*staking.StakingTransaction, common.Hash, uint64, uint64) { blockHash, blockNumber, txIndex := ReadTxLookupEntry(db, hash) if blockHash == (common.Hash{}) { return nil, common.Hash{}, 0, 0 @@ -163,13 +167,13 @@ func ReadStakingTransaction(db DatabaseReader, hash common.Hash) (*staking.Staki // ReadReceipt retrieves a specific transaction receipt from the database, along with // its added positional metadata. -func ReadReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Hash, uint64, uint64) { +func ReadReceipt(db ethdb.Reader, hash common.Hash, config *params.ChainConfig) (*types.Receipt, common.Hash, uint64, uint64) { blockHash, blockNumber, receiptIndex := ReadTxLookupEntry(db, hash) if blockHash == (common.Hash{}) { return nil, common.Hash{}, 0, 0 } - receipts := ReadReceipts(db, blockHash, blockNumber) + receipts := ReadReceipts(db, blockHash, blockNumber, nil) if len(receipts) <= int(receiptIndex) { utils.Logger().Error(). Uint64("number", blockNumber). @@ -183,13 +187,13 @@ func ReadReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Ha // ReadBloomBits retrieves the compressed bloom bit vector belonging to the given // section and bit index from the. -func ReadBloomBits(db DatabaseReader, bit uint, section uint64, head common.Hash) ([]byte, error) { +func ReadBloomBits(db ethdb.KeyValueReader, bit uint, section uint64, head common.Hash) ([]byte, error) { return db.Get(bloomBitsKey(bit, section, head)) } // WriteBloomBits stores the compressed bloom bits vector belonging to the given // section and bit index. -func WriteBloomBits(db DatabaseWriter, bit uint, section uint64, head common.Hash, bits []byte) error { +func WriteBloomBits(db ethdb.KeyValueWriter, bit uint, section uint64, head common.Hash, bits []byte) error { if err := db.Put(bloomBitsKey(bit, section, head), bits); err != nil { utils.Logger().Error().Err(err).Msg("Failed to store bloom bits") return err @@ -247,7 +251,7 @@ func DeleteCxLookupEntry(db DatabaseDeleter, hash common.Hash) error { // ReadCXReceipt retrieves a specific transaction from the database, along with // its added positional metadata. -func ReadCXReceipt(db DatabaseReader, hash common.Hash) (*types.CXReceipt, common.Hash, uint64, uint64) { +func ReadCXReceipt(db ethdb.Reader, hash common.Hash) (*types.CXReceipt, common.Hash, uint64, uint64) { blockHash, blockNumber, cxIndex := ReadCxLookupEntry(db, hash) if blockHash == (common.Hash{}) { return nil, common.Hash{}, 0, 0 @@ -272,3 +276,57 @@ func ReadCXReceipt(db DatabaseReader, hash common.Hash) (*types.CXReceipt, commo } return cx, blockHash, blockNumber, cxIndex } + +// writeTxLookupEntry stores a positional metadata for a transaction, +// enabling hash based transaction and receipt lookups. +func writeTxLookupEntry(db ethdb.KeyValueWriter, hash common.Hash, numberBytes []byte) { + if err := db.Put(txLookupKey(hash), numberBytes); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store transaction lookup entry") + } +} + +// WriteTxLookupEntries is identical to WriteTxLookupEntry, but it works on +// a list of hashes +func WriteTxLookupEntries(db ethdb.KeyValueWriter, number uint64, hashes []common.Hash) { + numberBytes := new(big.Int).SetUint64(number).Bytes() + for _, hash := range hashes { + writeTxLookupEntry(db, hash, numberBytes) + } +} + +// WriteTxLookupEntriesByBlock stores a positional metadata for every transaction from +// a block, enabling hash based transaction and receipt lookups. +func WriteTxLookupEntriesByBlock(db ethdb.KeyValueWriter, block *types.Block) { + numberBytes := block.Number().Bytes() + for _, tx := range block.Transactions() { + writeTxLookupEntry(db, tx.Hash(), numberBytes) + } +} + +// DeleteTxLookupEntries removes all transaction lookups for a given block. +func DeleteTxLookupEntries(db ethdb.KeyValueWriter, hashes []common.Hash) { + for _, hash := range hashes { + DeleteTxLookupEntry(db, hash) + } +} + +// DeleteBloombits removes all compressed bloom bits vector belonging to the +// given section range and bit index. +func DeleteBloombits(db ethdb.Database, bit uint, from uint64, to uint64) { + start, end := bloomBitsKey(bit, from, common.Hash{}), bloomBitsKey(bit, to, common.Hash{}) + it := db.NewIterator(nil, start) + defer it.Release() + + for it.Next() { + if bytes.Compare(it.Key(), end) >= 0 { + break + } + if len(it.Key()) != len(bloomBitsPrefix)+2+8+32 { + continue + } + db.Delete(it.Key()) + } + if it.Error() != nil { + utils.Logger().Error().Err(it.Error()).Msg("Failed to delete bloom bits") + } +} diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index 9001b7894c..6fef26d09e 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -18,81 +18,177 @@ package rawdb import ( "encoding/json" - "errors" + "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/utils" ) // ReadDatabaseVersion retrieves the version number of the database. -func ReadDatabaseVersion(db DatabaseReader) int { - var version int +func ReadDatabaseVersion(db ethdb.KeyValueReader) *uint64 { + var version uint64 - enc, _ := db.Get(databaseVerisionKey) - rlp.DecodeBytes(enc, &version) + enc, _ := db.Get(databaseVersionKey) + if len(enc) == 0 { + return nil + } + if err := rlp.DecodeBytes(enc, &version); err != nil { + return nil + } - return version + return &version } // WriteDatabaseVersion stores the version number of the database -func WriteDatabaseVersion(db DatabaseWriter, version int) error { - enc, _ := rlp.EncodeToBytes(version) - if err := db.Put(databaseVerisionKey, enc); err != nil { +func WriteDatabaseVersion(db ethdb.KeyValueWriter, version uint64) { + enc, err := rlp.EncodeToBytes(version) + if err != nil { + utils.Logger().Error().Err(err).Msg("Failed to encode database version") + } + if err = db.Put(databaseVersionKey, enc); err != nil { utils.Logger().Error().Err(err).Msg("Failed to store the database version") - return err } - return nil } // ReadChainConfig retrieves the consensus settings based on the given genesis hash. -func ReadChainConfig(db DatabaseReader, hash common.Hash) *params.ChainConfig { +func ReadChainConfig(db ethdb.KeyValueReader, hash common.Hash) *params.ChainConfig { data, _ := db.Get(configKey(hash)) if len(data) == 0 { return nil } var config params.ChainConfig if err := json.Unmarshal(data, &config); err != nil { - utils.Logger().Error().Err(err).Str("hash", hash.Hex()).Msg("Invalid chain config JSON") + utils.Logger().Error().Err(err).Interface("hash", hash).Msg("Invalid chain config JSON") return nil } return &config } // WriteChainConfig writes the chain config settings to the database. -func WriteChainConfig(db DatabaseWriter, hash common.Hash, cfg *params.ChainConfig) error { +func WriteChainConfig(db ethdb.KeyValueWriter, hash common.Hash, cfg *params.ChainConfig) { if cfg == nil { - return errors.New("nil config") + return } data, err := json.Marshal(cfg) if err != nil { utils.Logger().Error().Err(err).Msg("Failed to JSON encode chain config") - return err } if err := db.Put(configKey(hash), data); err != nil { utils.Logger().Error().Err(err).Msg("Failed to store chain config") - return err } - return nil } -// ReadPreimage retrieves a single preimage of the provided hash. -func ReadPreimage(db DatabaseReader, hash common.Hash) []byte { - data, _ := db.Get(preimageKey(hash)) +// ReadGenesisStateSpec retrieves the genesis state specification based on the +// given genesis (block-)hash. +func ReadGenesisStateSpec(db ethdb.KeyValueReader, blockhash common.Hash) []byte { + data, _ := db.Get(genesisStateSpecKey(blockhash)) return data } -// WritePreimages writes the provided set of preimages to the database. `number` is the -// current block number, and is used for debug messages only. -func WritePreimages(db DatabaseWriter, number uint64, preimages map[common.Hash][]byte) error { - for hash, preimage := range preimages { - if err := db.Put(preimageKey(hash), preimage); err != nil { - utils.Logger().Error().Err(err).Msg("Failed to store trie preimage") - return err - } - } - preimageCounter.Inc(int64(len(preimages))) - preimageHitCounter.Inc(int64(len(preimages))) - return nil +// WriteGenesisStateSpec writes the genesis state specification into the disk. +func WriteGenesisStateSpec(db ethdb.KeyValueWriter, blockhash common.Hash, data []byte) { + if err := db.Put(genesisStateSpecKey(blockhash), data); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store genesis state") + } +} + +// crashList is a list of unclean-shutdown-markers, for rlp-encoding to the +// database +type crashList struct { + Discarded uint64 // how many ucs have we deleted + Recent []uint64 // unix timestamps of 10 latest unclean shutdowns +} + +const crashesToKeep = 10 + +// PushUncleanShutdownMarker appends a new unclean shutdown marker and returns +// the previous data +// - a list of timestamps +// - a count of how many old unclean-shutdowns have been discarded +// This function is NOT used, just ported over from the Ethereum +func PushUncleanShutdownMarker(db ethdb.KeyValueStore) ([]uint64, uint64, error) { + var uncleanShutdowns crashList + // Read old data + if data, err := db.Get(uncleanShutdownKey); err != nil { + utils.Logger().Warn().Err(err).Msg("Error reading unclean shutdown markers") + } else if err := rlp.DecodeBytes(data, &uncleanShutdowns); err != nil { + return nil, 0, err + } + var discarded = uncleanShutdowns.Discarded + var previous = make([]uint64, len(uncleanShutdowns.Recent)) + copy(previous, uncleanShutdowns.Recent) + // Add a new (but cap it) + uncleanShutdowns.Recent = append(uncleanShutdowns.Recent, uint64(time.Now().Unix())) + if count := len(uncleanShutdowns.Recent); count > crashesToKeep+1 { + numDel := count - (crashesToKeep + 1) + uncleanShutdowns.Recent = uncleanShutdowns.Recent[numDel:] + uncleanShutdowns.Discarded += uint64(numDel) + } + // And save it again + data, _ := rlp.EncodeToBytes(uncleanShutdowns) + if err := db.Put(uncleanShutdownKey, data); err != nil { + utils.Logger().Warn().Err(err).Msg("Failed to write unclean-shutdown marker") + return nil, 0, err + } + return previous, discarded, nil +} + +// PopUncleanShutdownMarker removes the last unclean shutdown marker +// This function is NOT used, just ported over from the Ethereum +func PopUncleanShutdownMarker(db ethdb.KeyValueStore) { + var uncleanShutdowns crashList + // Read old data + if data, err := db.Get(uncleanShutdownKey); err != nil { + utils.Logger().Warn().Err(err).Msg("Error reading unclean shutdown markers") + } else if err := rlp.DecodeBytes(data, &uncleanShutdowns); err != nil { + utils.Logger().Error().Err(err).Msg("Error decoding unclean shutdown markers") // Should mos def _not_ happen + } + if l := len(uncleanShutdowns.Recent); l > 0 { + uncleanShutdowns.Recent = uncleanShutdowns.Recent[:l-1] + } + data, _ := rlp.EncodeToBytes(uncleanShutdowns) + if err := db.Put(uncleanShutdownKey, data); err != nil { + utils.Logger().Warn().Err(err).Msg("Failed to clear unclean-shutdown marker") + } +} + +// UpdateUncleanShutdownMarker updates the last marker's timestamp to now. +// This function is NOT used, just ported over from the Ethereum +func UpdateUncleanShutdownMarker(db ethdb.KeyValueStore) { + var uncleanShutdowns crashList + // Read old data + if data, err := db.Get(uncleanShutdownKey); err != nil { + utils.Logger().Warn().Err(err).Msg("Error reading unclean shutdown markers") + } else if err := rlp.DecodeBytes(data, &uncleanShutdowns); err != nil { + utils.Logger().Warn().Err(err).Msg("Error decoding unclean shutdown markers") + } + // This shouldn't happen because we push a marker on Backend instantiation + count := len(uncleanShutdowns.Recent) + if count == 0 { + utils.Logger().Warn().Msg("No unclean shutdown marker to update") + return + } + uncleanShutdowns.Recent[count-1] = uint64(time.Now().Unix()) + data, _ := rlp.EncodeToBytes(uncleanShutdowns) + if err := db.Put(uncleanShutdownKey, data); err != nil { + utils.Logger().Warn().Err(err).Msg("Failed to write unclean-shutdown marker") + } +} + +// ReadTransitionStatus retrieves the eth2 transition status from the database +// This function is NOT used, just ported over from the Ethereum +func ReadTransitionStatus(db ethdb.KeyValueReader) []byte { + data, _ := db.Get(transitionStatusKey) + return data +} + +// WriteTransitionStatus stores the eth2 transition status to the database +// This function is NOT used, just ported over from the Ethereum +func WriteTransitionStatus(db ethdb.KeyValueWriter, data []byte) { + if err := db.Put(transitionStatusKey, data); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store the eth2 transition status") + } } diff --git a/core/rawdb/accessors_offchain.go b/core/rawdb/accessors_offchain.go index 5324aa0e50..dd43299034 100644 --- a/core/rawdb/accessors_offchain.go +++ b/core/rawdb/accessors_offchain.go @@ -189,7 +189,7 @@ func DeleteValidatorSnapshot(db DatabaseDeleter, addr common.Address, epoch *big } func IteratorValidatorSnapshot(iterator DatabaseIterator, cb func(addr common.Address, epoch *big.Int) bool) (minKey []byte, maxKey []byte) { - iter := iterator.NewIteratorWithPrefix(validatorSnapshotPrefix) + iter := iterator.NewIterator(validatorSnapshotPrefix, nil) defer iter.Release() minKey = validatorSnapshotPrefix @@ -211,7 +211,7 @@ func IteratorValidatorSnapshot(iterator DatabaseIterator, cb func(addr common.Ad func IteratorCXReceipt(iterator DatabaseIterator, cb func(it ethdb.Iterator, shardID uint32, number uint64, hash common.Hash) bool) { preifxKey := cxReceiptPrefix - iter := iterator.NewIteratorWithPrefix(preifxKey) + iter := iterator.NewIterator(preifxKey, nil) defer iter.Release() shardOffset := len(preifxKey) numberOffset := shardOffset + 4 @@ -231,7 +231,7 @@ func IteratorCXReceipt(iterator DatabaseIterator, cb func(it ethdb.Iterator, sha func IteratorCXReceiptsProofSpent(iterator DatabaseIterator, cb func(it ethdb.Iterator, shardID uint32, number uint64) bool) { preifxKey := cxReceiptSpentPrefix - iter := iterator.NewIteratorWithPrefix(preifxKey) + iter := iterator.NewIterator(preifxKey, nil) defer iter.Release() shardOffset := len(preifxKey) numberOffset := shardOffset + 4 @@ -248,7 +248,7 @@ func IteratorCXReceiptsProofSpent(iterator DatabaseIterator, cb func(it ethdb.It } func IteratorValidatorStats(iterator DatabaseIterator, cb func(it ethdb.Iterator, addr common.Address) bool) { preifxKey := validatorStatsPrefix - iter := iterator.NewIteratorWithPrefix(preifxKey) + iter := iterator.NewIterator(preifxKey, nil) defer iter.Release() addrOffset := len(preifxKey) @@ -263,7 +263,7 @@ func IteratorValidatorStats(iterator DatabaseIterator, cb func(it ethdb.Iterator } func IteratorDelegatorDelegations(iterator DatabaseIterator, cb func(it ethdb.Iterator, delegator common.Address) bool) { preifxKey := delegatorValidatorListPrefix - iter := iterator.NewIteratorWithPrefix(preifxKey) + iter := iterator.NewIterator(preifxKey, nil) defer iter.Release() addrOffset := len(preifxKey) diff --git a/core/rawdb/accessors_snapshot.go b/core/rawdb/accessors_snapshot.go new file mode 100644 index 0000000000..dd2119f8d8 --- /dev/null +++ b/core/rawdb/accessors_snapshot.go @@ -0,0 +1,210 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "encoding/binary" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/harmony-one/harmony/internal/utils" +) + +// ReadSnapshotDisabled retrieves if the snapshot maintenance is disabled. +func ReadSnapshotDisabled(db ethdb.KeyValueReader) bool { + disabled, _ := db.Has(snapshotDisabledKey) + return disabled +} + +// WriteSnapshotDisabled stores the snapshot pause flag. +func WriteSnapshotDisabled(db ethdb.KeyValueWriter) { + if err := db.Put(snapshotDisabledKey, []byte("42")); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store snapshot disabled flag") + } +} + +// DeleteSnapshotDisabled deletes the flag keeping the snapshot maintenance disabled. +func DeleteSnapshotDisabled(db ethdb.KeyValueWriter) { + if err := db.Delete(snapshotDisabledKey); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to remove snapshot disabled flag") + } +} + +// ReadSnapshotRoot retrieves the root of the block whose state is contained in +// the persisted snapshot. +func ReadSnapshotRoot(db ethdb.KeyValueReader) common.Hash { + data, _ := db.Get(SnapshotRootKey) + if len(data) != common.HashLength { + return common.Hash{} + } + return common.BytesToHash(data) +} + +// WriteSnapshotRoot stores the root of the block whose state is contained in +// the persisted snapshot. +func WriteSnapshotRoot(db ethdb.KeyValueWriter, root common.Hash) { + if err := db.Put(SnapshotRootKey, root[:]); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store snapshot root") + } +} + +// DeleteSnapshotRoot deletes the hash of the block whose state is contained in +// the persisted snapshot. Since snapshots are not immutable, this method can +// be used during updates, so a crash or failure will mark the entire snapshot +// invalid. +func DeleteSnapshotRoot(db ethdb.KeyValueWriter) { + if err := db.Delete(SnapshotRootKey); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to remove snapshot root") + } +} + +// ReadAccountSnapshot retrieves the snapshot entry of an account trie leaf. +func ReadAccountSnapshot(db ethdb.KeyValueReader, hash common.Hash) []byte { + data, _ := db.Get(accountSnapshotKey(hash)) + return data +} + +// WriteAccountSnapshot stores the snapshot entry of an account trie leaf. +func WriteAccountSnapshot(db ethdb.KeyValueWriter, hash common.Hash, entry []byte) { + if err := db.Put(accountSnapshotKey(hash), entry); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store account snapshot") + } +} + +// DeleteAccountSnapshot removes the snapshot entry of an account trie leaf. +func DeleteAccountSnapshot(db ethdb.KeyValueWriter, hash common.Hash) { + if err := db.Delete(accountSnapshotKey(hash)); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to delete account snapshot") + } +} + +// ReadStorageSnapshot retrieves the snapshot entry of an storage trie leaf. +func ReadStorageSnapshot(db ethdb.KeyValueReader, accountHash, storageHash common.Hash) []byte { + data, _ := db.Get(storageSnapshotKey(accountHash, storageHash)) + return data +} + +// WriteStorageSnapshot stores the snapshot entry of an storage trie leaf. +func WriteStorageSnapshot(db ethdb.KeyValueWriter, accountHash, storageHash common.Hash, entry []byte) { + if err := db.Put(storageSnapshotKey(accountHash, storageHash), entry); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store storage snapshot") + } +} + +// DeleteStorageSnapshot removes the snapshot entry of an storage trie leaf. +func DeleteStorageSnapshot(db ethdb.KeyValueWriter, accountHash, storageHash common.Hash) { + if err := db.Delete(storageSnapshotKey(accountHash, storageHash)); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to delete storage snapshot") + } +} + +// IterateStorageSnapshots returns an iterator for walking the entire storage +// space of a specific account. +func IterateStorageSnapshots(db ethdb.Iteratee, accountHash common.Hash) ethdb.Iterator { + return NewKeyLengthIterator(db.NewIterator(storageSnapshotsKey(accountHash), nil), len(SnapshotStoragePrefix)+2*common.HashLength) +} + +// ReadSnapshotJournal retrieves the serialized in-memory diff layers saved at +// the last shutdown. The blob is expected to be max a few 10s of megabytes. +func ReadSnapshotJournal(db ethdb.KeyValueReader) []byte { + data, _ := db.Get(snapshotJournalKey) + return data +} + +// WriteSnapshotJournal stores the serialized in-memory diff layers to save at +// shutdown. The blob is expected to be max a few 10s of megabytes. +func WriteSnapshotJournal(db ethdb.KeyValueWriter, journal []byte) { + if err := db.Put(snapshotJournalKey, journal); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store snapshot journal") + } +} + +// DeleteSnapshotJournal deletes the serialized in-memory diff layers saved at +// the last shutdown +func DeleteSnapshotJournal(db ethdb.KeyValueWriter) { + if err := db.Delete(snapshotJournalKey); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to remove snapshot journal") + } +} + +// ReadSnapshotGenerator retrieves the serialized snapshot generator saved at +// the last shutdown. +func ReadSnapshotGenerator(db ethdb.KeyValueReader) []byte { + data, _ := db.Get(snapshotGeneratorKey) + return data +} + +// WriteSnapshotGenerator stores the serialized snapshot generator to save at +// shutdown. +func WriteSnapshotGenerator(db ethdb.KeyValueWriter, generator []byte) { + if err := db.Put(snapshotGeneratorKey, generator); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store snapshot generator") + } +} + +// DeleteSnapshotGenerator deletes the serialized snapshot generator saved at +// the last shutdown +func DeleteSnapshotGenerator(db ethdb.KeyValueWriter) { + if err := db.Delete(snapshotGeneratorKey); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to remove snapshot generator") + } +} + +// ReadSnapshotRecoveryNumber retrieves the block number of the last persisted +// snapshot layer. +func ReadSnapshotRecoveryNumber(db ethdb.KeyValueReader) *uint64 { + data, _ := db.Get(snapshotRecoveryKey) + if len(data) == 0 { + return nil + } + if len(data) != 8 { + return nil + } + number := binary.BigEndian.Uint64(data) + return &number +} + +// WriteSnapshotRecoveryNumber stores the block number of the last persisted +// snapshot layer. +func WriteSnapshotRecoveryNumber(db ethdb.KeyValueWriter, number uint64) { + var buf [8]byte + binary.BigEndian.PutUint64(buf[:], number) + if err := db.Put(snapshotRecoveryKey, buf[:]); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store snapshot recovery number") + } +} + +// DeleteSnapshotRecoveryNumber deletes the block number of the last persisted +// snapshot layer. +func DeleteSnapshotRecoveryNumber(db ethdb.KeyValueWriter) { + if err := db.Delete(snapshotRecoveryKey); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to remove snapshot recovery number") + } +} + +// ReadSnapshotSyncStatus retrieves the serialized sync status saved at shutdown. +func ReadSnapshotSyncStatus(db ethdb.KeyValueReader) []byte { + data, _ := db.Get(snapshotSyncStatusKey) + return data +} + +// WriteSnapshotSyncStatus stores the serialized sync status to save at shutdown. +func WriteSnapshotSyncStatus(db ethdb.KeyValueWriter, status []byte) { + if err := db.Put(snapshotSyncStatusKey, status); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store snapshot sync status") + } +} diff --git a/core/rawdb/accessors_state.go b/core/rawdb/accessors_state.go new file mode 100644 index 0000000000..351928fcd5 --- /dev/null +++ b/core/rawdb/accessors_state.go @@ -0,0 +1,95 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/harmony-one/harmony/internal/utils" +) + +// ReadPreimage retrieves a single preimage of the provided hash. +func ReadPreimage(db ethdb.KeyValueReader, hash common.Hash) []byte { + data, _ := db.Get(preimageKey(hash)) + return data +} + +// WritePreimages writes the provided set of preimages to the database. +func WritePreimages(db ethdb.KeyValueWriter, preimages map[common.Hash][]byte) error { + for hash, preimage := range preimages { + if err := db.Put(preimageKey(hash), preimage); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store trie preimage") + } + } + preimageCounter.Inc(int64(len(preimages))) + preimageHitCounter.Inc(int64(len(preimages))) + return nil +} + +// ReadCode retrieves the contract code of the provided code hash. +func ReadCode(db ethdb.KeyValueReader, hash common.Hash) []byte { + // Try with the prefixed code scheme first, if not then try with legacy + // scheme. + data := ReadCodeWithPrefix(db, hash) + if len(data) != 0 { + return data + } + data, _ = db.Get(hash.Bytes()) + return data +} + +// ReadCodeWithPrefix retrieves the contract code of the provided code hash. +// The main difference between this function and ReadCode is this function +// will only check the existence with latest scheme(with prefix). +func ReadCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) []byte { + data, _ := db.Get(codeKey(hash)) + return data +} + +// HasCode checks if the contract code corresponding to the +// provided code hash is present in the db. +func HasCode(db ethdb.KeyValueReader, hash common.Hash) bool { + // Try with the prefixed code scheme first, if not then try with legacy + // scheme. + if ok := HasCodeWithPrefix(db, hash); ok { + return true + } + ok, _ := db.Has(hash.Bytes()) + return ok +} + +// HasCodeWithPrefix checks if the contract code corresponding to the +// provided code hash is present in the db. This function will only check +// presence using the prefix-scheme. +func HasCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) bool { + ok, _ := db.Has(codeKey(hash)) + return ok +} + +// WriteCode writes the provided contract code database. +func WriteCode(db ethdb.KeyValueWriter, hash common.Hash, code []byte) { + if err := db.Put(codeKey(hash), code); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store contract code") + } +} + +// DeleteCode deletes the specified contract code from the database. +func DeleteCode(db ethdb.KeyValueWriter, hash common.Hash) { + if err := db.Delete(codeKey(hash)); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to delete contract code") + } +} diff --git a/core/rawdb/accessors_sync.go b/core/rawdb/accessors_sync.go new file mode 100644 index 0000000000..d5f4f34e4e --- /dev/null +++ b/core/rawdb/accessors_sync.go @@ -0,0 +1,80 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "bytes" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" + "github.com/harmony-one/harmony/internal/utils" +) + +// ReadSkeletonSyncStatus retrieves the serialized sync status saved at shutdown. +func ReadSkeletonSyncStatus(db ethdb.KeyValueReader) []byte { + data, _ := db.Get(skeletonSyncStatusKey) + return data +} + +// WriteSkeletonSyncStatus stores the serialized sync status to save at shutdown. +func WriteSkeletonSyncStatus(db ethdb.KeyValueWriter, status []byte) { + if err := db.Put(skeletonSyncStatusKey, status); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store skeleton sync status") + } +} + +// DeleteSkeletonSyncStatus deletes the serialized sync status saved at the last +// shutdown +func DeleteSkeletonSyncStatus(db ethdb.KeyValueWriter) { + if err := db.Delete(skeletonSyncStatusKey); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to remove skeleton sync status") + } +} + +// ReadSkeletonHeader retrieves a block header from the skeleton sync store, +func ReadSkeletonHeader(db ethdb.KeyValueReader, number uint64) *types.Header { + data, _ := db.Get(skeletonHeaderKey(number)) + if len(data) == 0 { + return nil + } + header := new(types.Header) + if err := rlp.Decode(bytes.NewReader(data), header); err != nil { + utils.Logger().Error().Err(err).Uint64("number", number).Msg("Invalid skeleton header RLP") + return nil + } + return header +} + +// WriteSkeletonHeader stores a block header into the skeleton sync store. +func WriteSkeletonHeader(db ethdb.KeyValueWriter, header *types.Header) { + data, err := rlp.EncodeToBytes(header) + if err != nil { + utils.Logger().Error().Err(err).Msg("Failed to RLP encode header") + } + key := skeletonHeaderKey(header.Number.Uint64()) + if err := db.Put(key, data); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store skeleton header") + } +} + +// DeleteSkeletonHeader removes all block header data associated with a hash. +func DeleteSkeletonHeader(db ethdb.KeyValueWriter, number uint64) { + if err := db.Delete(skeletonHeaderKey(number)); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to delete skeleton header") + } +} diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go new file mode 100644 index 0000000000..b551cfc452 --- /dev/null +++ b/core/rawdb/accessors_trie.go @@ -0,0 +1,263 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package rawdb + +import ( + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/harmony-one/harmony/internal/utils" + "golang.org/x/crypto/sha3" +) + +// HashScheme is the legacy hash-based state scheme with which trie nodes are +// stored in the disk with node hash as the database key. The advantage of this +// scheme is that different versions of trie nodes can be stored in disk, which +// is very beneficial for constructing archive nodes. The drawback is it will +// store different trie nodes on the same path to different locations on the disk +// with no data locality, and it's unfriendly for designing state pruning. +// +// Now this scheme is still kept for backward compatibility, and it will be used +// for archive node and some other tries(e.g. light trie). +const HashScheme = "hashScheme" + +// PathScheme is the new path-based state scheme with which trie nodes are stored +// in the disk with node path as the database key. This scheme will only store one +// version of state data in the disk, which means that the state pruning operation +// is native. At the same time, this scheme will put adjacent trie nodes in the same +// area of the disk with good data locality property. But this scheme needs to rely +// on extra state diffs to survive deep reorg. +const PathScheme = "pathScheme" + +// nodeHasher used to derive the hash of trie node. +type nodeHasher struct{ sha crypto.KeccakState } + +var hasherPool = sync.Pool{ + New: func() interface{} { return &nodeHasher{sha: sha3.NewLegacyKeccak256().(crypto.KeccakState)} }, +} + +func newNodeHasher() *nodeHasher { return hasherPool.Get().(*nodeHasher) } +func returnHasherToPool(h *nodeHasher) { hasherPool.Put(h) } + +func (h *nodeHasher) hashData(data []byte) (n common.Hash) { + h.sha.Reset() + h.sha.Write(data) + h.sha.Read(n[:]) + return n +} + +// ReadAccountTrieNode retrieves the account trie node and the associated node +// hash with the specified node path. +func ReadAccountTrieNode(db ethdb.KeyValueReader, path []byte) ([]byte, common.Hash) { + data, err := db.Get(accountTrieNodeKey(path)) + if err != nil { + return nil, common.Hash{} + } + hasher := newNodeHasher() + defer returnHasherToPool(hasher) + return data, hasher.hashData(data) +} + +// HasAccountTrieNode checks the account trie node presence with the specified +// node path and the associated node hash. +func HasAccountTrieNode(db ethdb.KeyValueReader, path []byte, hash common.Hash) bool { + data, err := db.Get(accountTrieNodeKey(path)) + if err != nil { + return false + } + hasher := newNodeHasher() + defer returnHasherToPool(hasher) + return hasher.hashData(data) == hash +} + +// WriteAccountTrieNode writes the provided account trie node into database. +func WriteAccountTrieNode(db ethdb.KeyValueWriter, path []byte, node []byte) { + if err := db.Put(accountTrieNodeKey(path), node); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store account trie node") + } +} + +// DeleteAccountTrieNode deletes the specified account trie node from the database. +func DeleteAccountTrieNode(db ethdb.KeyValueWriter, path []byte) { + if err := db.Delete(accountTrieNodeKey(path)); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to delete account trie node") + } +} + +// ReadStorageTrieNode retrieves the storage trie node and the associated node +// hash with the specified node path. +func ReadStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte) ([]byte, common.Hash) { + data, err := db.Get(storageTrieNodeKey(accountHash, path)) + if err != nil { + return nil, common.Hash{} + } + hasher := newNodeHasher() + defer returnHasherToPool(hasher) + return data, hasher.hashData(data) +} + +// HasStorageTrieNode checks the storage trie node presence with the provided +// node path and the associated node hash. +func HasStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte, hash common.Hash) bool { + data, err := db.Get(storageTrieNodeKey(accountHash, path)) + if err != nil { + return false + } + hasher := newNodeHasher() + defer returnHasherToPool(hasher) + return hasher.hashData(data) == hash +} + +// WriteStorageTrieNode writes the provided storage trie node into database. +func WriteStorageTrieNode(db ethdb.KeyValueWriter, accountHash common.Hash, path []byte, node []byte) { + if err := db.Put(storageTrieNodeKey(accountHash, path), node); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store storage trie node") + } +} + +// DeleteStorageTrieNode deletes the specified storage trie node from the database. +func DeleteStorageTrieNode(db ethdb.KeyValueWriter, accountHash common.Hash, path []byte) { + if err := db.Delete(storageTrieNodeKey(accountHash, path)); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to delete storage trie node") + } +} + +// ReadLegacyTrieNode retrieves the legacy trie node with the given +// associated node hash. +func ReadLegacyTrieNode(db ethdb.KeyValueReader, hash common.Hash) []byte { + data, err := db.Get(hash.Bytes()) + if err != nil { + return nil + } + return data +} + +// HasLegacyTrieNode checks if the trie node with the provided hash is present in db. +func HasLegacyTrieNode(db ethdb.KeyValueReader, hash common.Hash) bool { + ok, _ := db.Has(hash.Bytes()) + return ok +} + +// WriteLegacyTrieNode writes the provided legacy trie node to database. +func WriteLegacyTrieNode(db ethdb.KeyValueWriter, hash common.Hash, node []byte) { + if err := db.Put(hash.Bytes(), node); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store legacy trie node") + } +} + +// DeleteLegacyTrieNode deletes the specified legacy trie node from database. +func DeleteLegacyTrieNode(db ethdb.KeyValueWriter, hash common.Hash) { + if err := db.Delete(hash.Bytes()); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to delete legacy trie node") + } +} + +// HasTrieNode checks the trie node presence with the provided node info and +// the associated node hash. +func HasTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash, scheme string) bool { + switch scheme { + case HashScheme: + return HasLegacyTrieNode(db, hash) + case PathScheme: + if owner == (common.Hash{}) { + return HasAccountTrieNode(db, path, hash) + } + return HasStorageTrieNode(db, owner, path, hash) + default: + panic(fmt.Sprintf("Unknown scheme %v", scheme)) + } +} + +// ReadTrieNode retrieves the trie node from database with the provided node info +// and associated node hash. +// hashScheme-based lookup requires the following: +// - hash +// +// pathScheme-based lookup requires the following: +// - owner +// - path +func ReadTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash, scheme string) []byte { + switch scheme { + case HashScheme: + return ReadLegacyTrieNode(db, hash) + case PathScheme: + var ( + blob []byte + nHash common.Hash + ) + if owner == (common.Hash{}) { + blob, nHash = ReadAccountTrieNode(db, path) + } else { + blob, nHash = ReadStorageTrieNode(db, owner, path) + } + if nHash != hash { + return nil + } + return blob + default: + panic(fmt.Sprintf("Unknown scheme %v", scheme)) + } +} + +// WriteTrieNode writes the trie node into database with the provided node info +// and associated node hash. +// hashScheme-based lookup requires the following: +// - hash +// +// pathScheme-based lookup requires the following: +// - owner +// - path +func WriteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash, node []byte, scheme string) { + switch scheme { + case HashScheme: + WriteLegacyTrieNode(db, hash, node) + case PathScheme: + if owner == (common.Hash{}) { + WriteAccountTrieNode(db, path, node) + } else { + WriteStorageTrieNode(db, owner, path, node) + } + default: + panic(fmt.Sprintf("Unknown scheme %v", scheme)) + } +} + +// DeleteTrieNode deletes the trie node from database with the provided node info +// and associated node hash. +// hashScheme-based lookup requires the following: +// - hash +// +// pathScheme-based lookup requires the following: +// - owner +// - path +func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash, scheme string) { + switch scheme { + case HashScheme: + DeleteLegacyTrieNode(db, hash) + case PathScheme: + if owner == (common.Hash{}) { + DeleteAccountTrieNode(db, path) + } else { + DeleteStorageTrieNode(db, owner, path) + } + default: + panic(fmt.Sprintf("Unknown scheme %v", scheme)) + } +} diff --git a/core/rawdb/ancient_scheme.go b/core/rawdb/ancient_scheme.go new file mode 100644 index 0000000000..8b109c5a52 --- /dev/null +++ b/core/rawdb/ancient_scheme.go @@ -0,0 +1,55 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +// The list of table names of chain freezer. +// This variables is NOT used, just ported over from the Ethereum +const ( + // ChainFreezerHeaderTable indicates the name of the freezer header table. + ChainFreezerHeaderTable = "headers" + + // ChainFreezerHashTable indicates the name of the freezer canonical hash table. + ChainFreezerHashTable = "hashes" + + // ChainFreezerBodiesTable indicates the name of the freezer block body table. + ChainFreezerBodiesTable = "bodies" + + // ChainFreezerReceiptTable indicates the name of the freezer receipts table. + ChainFreezerReceiptTable = "receipts" + + // ChainFreezerDifficultyTable indicates the name of the freezer total difficulty table. + ChainFreezerDifficultyTable = "diffs" +) + +// chainFreezerNoSnappy configures whether compression is disabled for the ancient-tables. +// Hashes and difficulties don't compress well. +// This function is NOT used, just ported over from the Ethereum +var chainFreezerNoSnappy = map[string]bool{ + ChainFreezerHeaderTable: false, + ChainFreezerHashTable: true, + ChainFreezerBodiesTable: false, + ChainFreezerReceiptTable: false, + ChainFreezerDifficultyTable: true, +} + +// The list of identifiers of ancient stores. +var ( + chainFreezerName = "chain" // the folder name of chain segment ancient store. +) + +// freezers the collections of all builtin freezers. +var freezers = []string{chainFreezerName} diff --git a/core/rawdb/ancient_utils.go b/core/rawdb/ancient_utils.go new file mode 100644 index 0000000000..e4eaf559c2 --- /dev/null +++ b/core/rawdb/ancient_utils.go @@ -0,0 +1,91 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" +) + +type tableSize struct { + name string + size common.StorageSize +} + +// freezerInfo contains the basic information of the freezer. +type freezerInfo struct { + name string // The identifier of freezer + head uint64 // The number of last stored item in the freezer + tail uint64 // The number of first stored item in the freezer + sizes []tableSize // The storage size per table +} + +// count returns the number of stored items in the freezer. +func (info *freezerInfo) count() uint64 { + return info.head - info.tail + 1 +} + +// size returns the storage size of the entire freezer. +func (info *freezerInfo) size() common.StorageSize { + var total common.StorageSize + for _, table := range info.sizes { + total += table.size + } + return total +} + +// inspectFreezers inspects all freezers registered in the system. +// This function is NOT used, just ported over from the Ethereum +func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) { + var infos []freezerInfo + for _, freezer := range freezers { + switch freezer { + case chainFreezerName: + // Chain ancient store is a bit special. It's always opened along + // with the key-value store, inspect the chain store directly. + info := freezerInfo{name: freezer} + // Retrieve storage size of every contained table. + for table := range chainFreezerNoSnappy { + size, err := db.AncientSize(table) + if err != nil { + return nil, err + } + info.sizes = append(info.sizes, tableSize{name: table, size: common.StorageSize(size)}) + } + // Retrieve the number of last stored item + ancients, err := db.Ancients() + if err != nil { + return nil, err + } + info.head = ancients - 1 + + // Retrieve the number of first stored item + tail, err := db.Tail() + if err != nil { + return nil, err + } + info.tail = tail + infos = append(infos, info) + + default: + return nil, fmt.Errorf("unknown freezer, supported ones: %v", freezers) + } + } + return infos, nil +} diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go new file mode 100644 index 0000000000..048c1f948d --- /dev/null +++ b/core/rawdb/chain_iterator.go @@ -0,0 +1,358 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "runtime" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/prque" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/harmony-one/harmony/internal/utils" +) + +// InitDatabaseFromFreezer reinitializes an empty database from a previous batch +// of frozen ancient blocks. The method iterates over all the frozen blocks and +// injects into the database the block hash->number mappings. +// This function is NOT used, just ported over from the Ethereum +func InitDatabaseFromFreezer(db ethdb.Database) { + // If we can't access the freezer or it's empty, abort + frozen, err := db.Ancients() + if err != nil || frozen == 0 { + return + } + var ( + batch = db.NewBatch() + start = time.Now() + logged = start.Add(-7 * time.Second) // Unindex during import is fast, don't double log + hash common.Hash + ) + for i := uint64(0); i < frozen; { + // We read 100K hashes at a time, for a total of 3.2M + count := uint64(100_000) + if i+count > frozen { + count = frozen - i + } + data, err := db.AncientRange(ChainFreezerHashTable, i, count, 32*count) + if err != nil { + utils.Logger().Error().Err(err).Msg("Failed to init database from freezer") + } + for j, h := range data { + number := i + uint64(j) + hash = common.BytesToHash(h) + WriteHeaderNumber(batch, hash, number) + // If enough data was accumulated in memory or we're at the last block, dump to disk + if batch.ValueSize() > ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to write data to db") + } + batch.Reset() + } + } + i += uint64(len(data)) + // If we've spent too much time already, notify the user of what we're doing + if time.Since(logged) > 8*time.Second { + log.Info("Initializing database from freezer", "total", frozen, "number", i, "hash", hash, "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } + } + if err := batch.Write(); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to write data to db") + } + batch.Reset() + + WriteHeadHeaderHash(db, hash) + WriteHeadFastBlockHash(db, hash) + log.Info("Initialized database from freezer", "blocks", frozen, "elapsed", common.PrettyDuration(time.Since(start))) +} + +type blockTxHashes struct { + number uint64 + hashes []common.Hash +} + +// iterateTransactions iterates over all transactions in the (canon) block +// number(s) given, and yields the hashes on a channel. If there is a signal +// received from interrupt channel, the iteration will be aborted and result +// channel will be closed. +func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool, interrupt chan struct{}) chan *blockTxHashes { + // One thread sequentially reads data from db + type numberRlp struct { + number uint64 + rlp rlp.RawValue + } + if to == from { + return nil + } + threads := to - from + if cpus := runtime.NumCPU(); threads > uint64(cpus) { + threads = uint64(cpus) + } + var ( + rlpCh = make(chan *numberRlp, threads*2) // we send raw rlp over this channel + hashesCh = make(chan *blockTxHashes, threads*2) // send hashes over hashesCh + ) + // lookup runs in one instance + lookup := func() { + n, end := from, to + if reverse { + n, end = to-1, from-1 + } + defer close(rlpCh) + for n != end { + data := ReadCanonicalBodyRLP(db, n) + // Feed the block to the aggregator, or abort on interrupt + select { + case rlpCh <- &numberRlp{n, data}: + case <-interrupt: + return + } + if reverse { + n-- + } else { + n++ + } + } + } + // process runs in parallel + nThreadsAlive := int32(threads) + process := func() { + defer func() { + // Last processor closes the result channel + if atomic.AddInt32(&nThreadsAlive, -1) == 0 { + close(hashesCh) + } + }() + for data := range rlpCh { + var body types.Body + if err := rlp.DecodeBytes(data.rlp, &body); err != nil { + utils.Logger().Warn().Err(err).Uint64("block", data.number).Msg("Failed to decode block body") + return + } + var hashes []common.Hash + for _, tx := range body.Transactions { + hashes = append(hashes, tx.Hash()) + } + result := &blockTxHashes{ + hashes: hashes, + number: data.number, + } + // Feed the block to the aggregator, or abort on interrupt + select { + case hashesCh <- result: + case <-interrupt: + return + } + } + } + go lookup() // start the sequential db accessor + for i := 0; i < int(threads); i++ { + go process() + } + return hashesCh +} + +// indexTransactions creates txlookup indices of the specified block range. +// +// This function iterates canonical chain in reverse order, it has one main advantage: +// We can write tx index tail flag periodically even without the whole indexing +// procedure is finished. So that we can resume indexing procedure next time quickly. +// +// There is a passed channel, the whole procedure will be interrupted if any +// signal received. +// This function is NOT used, just ported over from the Ethereum +func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { + // short circuit for invalid range + if from >= to { + return + } + var ( + hashesCh = iterateTransactions(db, from, to, true, interrupt) + batch = db.NewBatch() + start = time.Now() + logged = start.Add(-7 * time.Second) + // Since we iterate in reverse, we expect the first number to come + // in to be [to-1]. Therefore, setting lastNum to means that the + // prqueue gap-evaluation will work correctly + lastNum = to + queue = prque.New[int64, *blockTxHashes](nil) + // for stats reporting + blocks, txs = 0, 0 + ) + for chanDelivery := range hashesCh { + // Push the delivery into the queue and process contiguous ranges. + // Since we iterate in reverse, so lower numbers have lower prio, and + // we can use the number directly as prio marker + queue.Push(chanDelivery, int64(chanDelivery.number)) + for !queue.Empty() { + // If the next available item is gapped, return + if _, priority := queue.Peek(); priority != int64(lastNum-1) { + break + } + // For testing + if hook != nil && !hook(lastNum-1) { + break + } + // Next block available, pop it off and index it + delivery := queue.PopItem() + lastNum = delivery.number + WriteTxLookupEntries(batch, delivery.number, delivery.hashes) + blocks++ + txs += len(delivery.hashes) + // If enough data was accumulated in memory or we're at the last block, dump to disk + if batch.ValueSize() > ethdb.IdealBatchSize { + WriteTxIndexTail(batch, lastNum) // Also write the tail here + if err := batch.Write(); err != nil { + utils.Logger().Error().Err(err).Msg("Failed writing batch to db") + return + } + batch.Reset() + } + // If we've spent too much time already, notify the user of what we're doing + if time.Since(logged) > 8*time.Second { + log.Info("Indexing transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } + } + } + // Flush the new indexing tail and the last committed data. It can also happen + // that the last batch is empty because nothing to index, but the tail has to + // be flushed anyway. + WriteTxIndexTail(batch, lastNum) + if err := batch.Write(); err != nil { + utils.Logger().Error().Err(err).Msg("Failed writing batch to db") + return + } + select { + case <-interrupt: + log.Debug("Transaction indexing interrupted", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) + default: + log.Debug("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) + } +} + +// IndexTransactions creates txlookup indices of the specified block range. The from +// is included while to is excluded. +// +// This function iterates canonical chain in reverse order, it has one main advantage: +// We can write tx index tail flag periodically even without the whole indexing +// procedure is finished. So that we can resume indexing procedure next time quickly. +// +// There is a passed channel, the whole procedure will be interrupted if any +// signal received. +func IndexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) { + indexTransactions(db, from, to, interrupt, nil) +} + +// indexTransactionsForTesting is the internal debug version with an additional hook. +func indexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { + indexTransactions(db, from, to, interrupt, hook) +} + +// unindexTransactions removes txlookup indices of the specified block range. +// +// There is a passed channel, the whole procedure will be interrupted if any +// signal received. +// This function is NOT used, just ported over from the Ethereum +func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { + // short circuit for invalid range + if from >= to { + return + } + var ( + hashesCh = iterateTransactions(db, from, to, false, interrupt) + batch = db.NewBatch() + start = time.Now() + logged = start.Add(-7 * time.Second) + // we expect the first number to come in to be [from]. Therefore, setting + // nextNum to from means that the prqueue gap-evaluation will work correctly + nextNum = from + queue = prque.New[int64, *blockTxHashes](nil) + // for stats reporting + blocks, txs = 0, 0 + ) + // Otherwise spin up the concurrent iterator and unindexer + for delivery := range hashesCh { + // Push the delivery into the queue and process contiguous ranges. + queue.Push(delivery, -int64(delivery.number)) + for !queue.Empty() { + // If the next available item is gapped, return + if _, priority := queue.Peek(); -priority != int64(nextNum) { + break + } + // For testing + if hook != nil && !hook(nextNum) { + break + } + delivery := queue.PopItem() + nextNum = delivery.number + 1 + DeleteTxLookupEntries(batch, delivery.hashes) + txs += len(delivery.hashes) + blocks++ + + // If enough data was accumulated in memory or we're at the last block, dump to disk + // A batch counts the size of deletion as '1', so we need to flush more + // often than that. + if blocks%1000 == 0 { + WriteTxIndexTail(batch, nextNum) + if err := batch.Write(); err != nil { + utils.Logger().Error().Err(err).Msg("Failed writing batch to db") + return + } + batch.Reset() + } + // If we've spent too much time already, notify the user of what we're doing + if time.Since(logged) > 8*time.Second { + log.Info("Unindexing transactions", "blocks", blocks, "txs", txs, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } + } + } + // Flush the new indexing tail and the last committed data. It can also happen + // that the last batch is empty because nothing to unindex, but the tail has to + // be flushed anyway. + WriteTxIndexTail(batch, nextNum) + if err := batch.Write(); err != nil { + utils.Logger().Error().Err(err).Msg("Failed writing batch to db") + return + } + select { + case <-interrupt: + log.Debug("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) + default: + log.Debug("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) + } +} + +// UnindexTransactions removes txlookup indices of the specified block range. +// The from is included while to is excluded. +// +// There is a passed channel, the whole procedure will be interrupted if any +// signal received. +func UnindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) { + unindexTransactions(db, from, to, interrupt, nil) +} + +// unindexTransactionsForTesting is the internal debug version with an additional hook. +func unindexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { + unindexTransactions(db, from, to, interrupt, hook) +} diff --git a/core/rawdb/database.go b/core/rawdb/database.go new file mode 100644 index 0000000000..5cdae00e38 --- /dev/null +++ b/core/rawdb/database.go @@ -0,0 +1,464 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "bytes" + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/leveldb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/log" + "github.com/harmony-one/harmony/internal/utils" + "github.com/olekukonko/tablewriter" +) + +var errNotSupported = errors.New("not supported") + +// convertLegacyFn takes a raw freezer entry in an older format and +// returns it in the new format. +type convertLegacyFn = func([]byte) ([]byte, error) + +// freezerdb is a database wrapper that enabled freezer data retrievals. +type freezerdb struct { + ancientRoot string + ethdb.KeyValueStore + ethdb.AncientStore +} + +// AncientDatadir returns the path of root ancient directory. +func (frdb *freezerdb) AncientDatadir() (string, error) { + return frdb.ancientRoot, nil +} + +// Close implements io.Closer, closing both the fast key-value store as well as +// the slow ancient tables. +func (frdb *freezerdb) Close() error { + var errs []error + if err := frdb.AncientStore.Close(); err != nil { + errs = append(errs, err) + } + if err := frdb.KeyValueStore.Close(); err != nil { + errs = append(errs, err) + } + if len(errs) != 0 { + return fmt.Errorf("%v", errs) + } + return nil +} + +// nofreezedb is a database wrapper that disables freezer data retrievals. +type nofreezedb struct { + ethdb.KeyValueStore +} + +// HasAncient returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) HasAncient(kind string, number uint64) (bool, error) { + return false, errNotSupported +} + +// Ancient returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) Ancient(kind string, number uint64) ([]byte, error) { + return nil, errNotSupported +} + +// AncientRange returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) AncientRange(kind string, start, max, maxByteSize uint64) ([][]byte, error) { + return nil, errNotSupported +} + +// Ancients returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) Ancients() (uint64, error) { + return 0, errNotSupported +} + +// Tail returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) Tail() (uint64, error) { + return 0, errNotSupported +} + +// AncientSize returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) AncientSize(kind string) (uint64, error) { + return 0, errNotSupported +} + +// ModifyAncients is not supported. +func (db *nofreezedb) ModifyAncients(func(ethdb.AncientWriteOp) error) (int64, error) { + return 0, errNotSupported +} + +// TruncateHead returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) TruncateHead(items uint64) error { + return errNotSupported +} + +// TruncateTail returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) TruncateTail(items uint64) error { + return errNotSupported +} + +// Sync returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) Sync() error { + return errNotSupported +} + +func (db *nofreezedb) ReadAncients(fn func(reader ethdb.AncientReaderOp) error) (err error) { + // Unlike other ancient-related methods, this method does not return + // errNotSupported when invoked. + // The reason for this is that the caller might want to do several things: + // 1. Check if something is in freezer, + // 2. If not, check leveldb. + // + // This will work, since the ancient-checks inside 'fn' will return errors, + // and the leveldb work will continue. + // + // If we instead were to return errNotSupported here, then the caller would + // have to explicitly check for that, having an extra clause to do the + // non-ancient operations. + return fn(db) +} + +// MigrateTable processes the entries in a given table in sequence +// converting them to a new format if they're of an old format. +func (db *nofreezedb) MigrateTable(kind string, convert convertLegacyFn) error { + return errNotSupported +} + +// AncientDatadir returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) AncientDatadir() (string, error) { + return "", errNotSupported +} + +// NewDatabase creates a high level database on top of a given key-value data +// store without a freezer moving immutable chain segments into cold storage. +func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { + return &nofreezedb{KeyValueStore: db} +} + +// resolveChainFreezerDir is a helper function which resolves the absolute path +// of chain freezer by considering backward compatibility. +// This function is NOT used, just ported over from the Ethereum +func resolveChainFreezerDir(ancient string) string { + // Check if the chain freezer is already present in the specified + // sub folder, if not then two possibilities: + // - chain freezer is not initialized + // - chain freezer exists in legacy location (root ancient folder) + freezer := path.Join(ancient, chainFreezerName) + if !common.FileExist(freezer) { + if !common.FileExist(ancient) { + // The entire ancient store is not initialized, still use the sub + // folder for initialization. + } else { + // Ancient root is already initialized, then we hold the assumption + // that chain freezer is also initialized and located in root folder. + // In this case fallback to legacy location. + freezer = ancient + log.Info("Found legacy ancient chain path", "location", ancient) + } + } + return freezer +} + +// NewMemoryDatabase creates an ephemeral in-memory key-value database without a +// freezer moving immutable chain segments into cold storage. +func NewMemoryDatabase() ethdb.Database { + return NewDatabase(memorydb.New()) +} + +// NewMemoryDatabaseWithCap creates an ephemeral in-memory key-value database +// with an initial starting capacity, but without a freezer moving immutable +// chain segments into cold storage. +func NewMemoryDatabaseWithCap(size int) ethdb.Database { + return NewDatabase(memorydb.NewWithCap(size)) +} + +// NewLevelDBDatabase creates a persistent key-value database without a freezer +// moving immutable chain segments into cold storage. +func NewLevelDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { + db, err := leveldb.New(file, cache, handles, namespace, readonly) + if err != nil { + return nil, err + } + log.Info("Using LevelDB as the backing database") + return NewDatabase(db), nil +} + +const ( + dbPebble = "pebble" + dbLeveldb = "leveldb" +) + +// hasPreexistingDb checks the given data directory whether a database is already +// instantiated at that location, and if so, returns the type of database (or the +// empty string). +func hasPreexistingDb(path string) string { + if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil { + return "" // No pre-existing db + } + if matches, err := filepath.Glob(filepath.Join(path, "OPTIONS*")); len(matches) > 0 || err != nil { + if err != nil { + panic(err) // only possible if the pattern is malformed + } + return dbPebble + } + return dbLeveldb +} + +// OpenOptions contains the options to apply when opening a database. +// OBS: If AncientsDirectory is empty, it indicates that no freezer is to be used. +type OpenOptions struct { + Type string // "leveldb" | "pebble" + Directory string // the datadir + AncientsDirectory string // the ancients-dir + Namespace string // the namespace for database relevant metrics + Cache int // the capacity(in megabytes) of the data caching + Handles int // number of files to be open simultaneously + ReadOnly bool +} + +// openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble. +// +// type == null type != null +// +---------------------------------------- +// db is non-existent | leveldb default | specified type +// db is existent | from db | specified type (if compatible) +func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) { + existingDb := hasPreexistingDb(o.Directory) + if len(existingDb) != 0 && len(o.Type) != 0 && o.Type != existingDb { + return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb) + } + if o.Type == dbPebble || existingDb == dbPebble { + if PebbleEnabled { + log.Info("Using pebble as the backing database") + return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) + } else { + return nil, errors.New("db.engine 'pebble' not supported on this platform") + } + } + if len(o.Type) != 0 && o.Type != dbLeveldb { + return nil, fmt.Errorf("unknown db.engine %v", o.Type) + } + log.Info("Using leveldb as the backing database") + // Use leveldb, either as default (no explicit choice), or pre-existing, or chosen explicitly + return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) +} + +type counter uint64 + +func (c counter) String() string { + return fmt.Sprintf("%d", c) +} + +func (c counter) Percentage(current uint64) string { + return fmt.Sprintf("%d", current*100/uint64(c)) +} + +// stat stores sizes and count for a parameter +type stat struct { + size common.StorageSize + count counter +} + +// Add size to the stat and increase the counter by 1 +func (s *stat) Add(size common.StorageSize) { + s.size += size + s.count++ +} + +func (s *stat) Size() string { + return s.size.String() +} + +func (s *stat) Count() string { + return s.count.String() +} + +// InspectDatabase traverses the entire database and checks the size +// of all different categories of data. +// This function is NOT used, just ported over from the Ethereum +func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { + it := db.NewIterator(keyPrefix, keyStart) + defer it.Release() + + var ( + count int64 + start = time.Now() + logged = time.Now() + + // Key-value store statistics + headers stat + bodies stat + receipts stat + tds stat + numHashPairings stat + hashNumPairings stat + tries stat + codes stat + txLookups stat + accountSnaps stat + storageSnaps stat + preimages stat + bloomBits stat + beaconHeaders stat + cliqueSnaps stat + + // Les statistic + chtTrieNodes stat + bloomTrieNodes stat + + // Meta- and unaccounted data + metadata stat + unaccounted stat + + // Totals + total common.StorageSize + ) + // Inspect key-value database first. + for it.Next() { + var ( + key = it.Key() + size = common.StorageSize(len(key) + len(it.Value())) + ) + total += size + switch { + case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): + headers.Add(size) + case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength): + bodies.Add(size) + case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): + receipts.Add(size) + case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): + tds.Add(size) + case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): + numHashPairings.Add(size) + case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): + hashNumPairings.Add(size) + case len(key) == common.HashLength: + tries.Add(size) + case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength: + codes.Add(size) + case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): + txLookups.Add(size) + case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength): + accountSnaps.Add(size) + case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength): + storageSnaps.Add(size) + case bytes.HasPrefix(key, PreimagePrefix) && len(key) == (len(PreimagePrefix)+common.HashLength): + preimages.Add(size) + case bytes.HasPrefix(key, configPrefix) && len(key) == (len(configPrefix)+common.HashLength): + metadata.Add(size) + case bytes.HasPrefix(key, genesisPrefix) && len(key) == (len(genesisPrefix)+common.HashLength): + metadata.Add(size) + case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): + bloomBits.Add(size) + case bytes.HasPrefix(key, BloomBitsIndexPrefix): + bloomBits.Add(size) + case bytes.HasPrefix(key, skeletonHeaderPrefix) && len(key) == (len(skeletonHeaderPrefix)+8): + beaconHeaders.Add(size) + case bytes.HasPrefix(key, CliqueSnapshotPrefix) && len(key) == 7+common.HashLength: + cliqueSnaps.Add(size) + case bytes.HasPrefix(key, ChtTablePrefix) || + bytes.HasPrefix(key, ChtIndexTablePrefix) || + bytes.HasPrefix(key, ChtPrefix): // Canonical hash trie + chtTrieNodes.Add(size) + case bytes.HasPrefix(key, BloomTrieTablePrefix) || + bytes.HasPrefix(key, BloomTrieIndexPrefix) || + bytes.HasPrefix(key, BloomTriePrefix): // Bloomtrie sub + bloomTrieNodes.Add(size) + default: + var accounted bool + for _, meta := range [][]byte{ + databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, headFinalizedBlockKey, + lastPivotKey, fastTrieProgressKey, snapshotDisabledKey, SnapshotRootKey, snapshotJournalKey, + snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, + uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey, + } { + if bytes.Equal(key, meta) { + metadata.Add(size) + accounted = true + break + } + } + if !accounted { + unaccounted.Add(size) + } + } + count++ + if count%1000 == 0 && time.Since(logged) > 8*time.Second { + log.Info("Inspecting database", "count", count, "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } + } + // Display the database statistic of key-value store. + stats := [][]string{ + {"Key-Value store", "Headers", headers.Size(), headers.Count()}, + {"Key-Value store", "Bodies", bodies.Size(), bodies.Count()}, + {"Key-Value store", "Receipt lists", receipts.Size(), receipts.Count()}, + {"Key-Value store", "Difficulties", tds.Size(), tds.Count()}, + {"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()}, + {"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()}, + {"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()}, + {"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()}, + {"Key-Value store", "Contract codes", codes.Size(), codes.Count()}, + {"Key-Value store", "Trie nodes", tries.Size(), tries.Count()}, + {"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()}, + {"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()}, + {"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()}, + {"Key-Value store", "Beacon sync headers", beaconHeaders.Size(), beaconHeaders.Count()}, + {"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()}, + {"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()}, + {"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()}, + {"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()}, + } + // Inspect all registered append-only file store then. + ancients, err := inspectFreezers(db) + if err != nil { + return err + } + for _, ancient := range ancients { + for _, table := range ancient.sizes { + stats = append(stats, []string{ + fmt.Sprintf("Ancient store (%s)", strings.Title(ancient.name)), + strings.Title(table.name), + table.size.String(), + fmt.Sprintf("%d", ancient.count()), + }) + } + total += ancient.size() + } + table := tablewriter.NewWriter(os.Stdout) + table.SetHeader([]string{"Database", "Category", "Size", "Items"}) + table.SetFooter([]string{"", "Total", total.String(), " "}) + table.AppendBulk(stats) + table.Render() + + if unaccounted.size > 0 { + utils.Logger().Error(). + Interface("size", unaccounted.size). + Interface("count", unaccounted.count). + Msg("Database contains unaccounted data") + } + return nil +} diff --git a/core/rawdb/database_test.go b/core/rawdb/database_test.go new file mode 100644 index 0000000000..a0d7b5ec66 --- /dev/null +++ b/core/rawdb/database_test.go @@ -0,0 +1,17 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb diff --git a/core/rawdb/databases_64bit.go b/core/rawdb/databases_64bit.go new file mode 100644 index 0000000000..139ce7d347 --- /dev/null +++ b/core/rawdb/databases_64bit.go @@ -0,0 +1,37 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +//go:build arm64 || amd64 + +package rawdb + +import ( + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/pebble" +) + +// Pebble is unsuported on 32bit architecture +const PebbleEnabled = true + +// NewPebbleDBDatabase creates a persistent key-value database without a freezer +// moving immutable chain segments into cold storage. +func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { + db, err := pebble.New(file, cache, handles, namespace, readonly) + if err != nil { + return nil, err + } + return NewDatabase(db), nil +} diff --git a/core/rawdb/databases_non64bit.go b/core/rawdb/databases_non64bit.go new file mode 100644 index 0000000000..b8ab2ecada --- /dev/null +++ b/core/rawdb/databases_non64bit.go @@ -0,0 +1,34 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build !(arm64 || amd64) + +package rawdb + +import ( + "errors" + + "github.com/ethereum/go-ethereum/ethdb" +) + +// Pebble is unsuported on 32bit architecture +const PebbleEnabled = false + +// NewPebbleDBDatabase creates a persistent key-value database without a freezer +// moving immutable chain segments into cold storage. +func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { + return nil, errors.New("pebble is not supported on this platform") +} diff --git a/core/rawdb/interfaces.go b/core/rawdb/interfaces.go index ac91ae55ae..76971edc60 100644 --- a/core/rawdb/interfaces.go +++ b/core/rawdb/interfaces.go @@ -27,13 +27,15 @@ type DatabaseReader interface { // DatabaseWriter wraps the Put method of a backing data store. type DatabaseWriter interface { Put(key []byte, value []byte) error + Delete(key []byte) error } // DatabaseDeleter wraps the Delete method of a backing data store. type DatabaseDeleter interface { + Put(key []byte, value []byte) error Delete(key []byte) error } type DatabaseIterator interface { - NewIteratorWithPrefix(prefix []byte) ethdb.Iterator + NewIterator(prefix []byte, start []byte) ethdb.Iterator } diff --git a/core/rawdb/key_length_iterator.go b/core/rawdb/key_length_iterator.go new file mode 100644 index 0000000000..d1c5af269a --- /dev/null +++ b/core/rawdb/key_length_iterator.go @@ -0,0 +1,47 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import "github.com/ethereum/go-ethereum/ethdb" + +// KeyLengthIterator is a wrapper for a database iterator that ensures only key-value pairs +// with a specific key length will be returned. +type KeyLengthIterator struct { + requiredKeyLength int + ethdb.Iterator +} + +// NewKeyLengthIterator returns a wrapped version of the iterator that will only return key-value +// pairs where keys with a specific key length will be returned. +func NewKeyLengthIterator(it ethdb.Iterator, keyLen int) ethdb.Iterator { + return &KeyLengthIterator{ + Iterator: it, + requiredKeyLength: keyLen, + } +} + +func (it *KeyLengthIterator) Next() bool { + // Return true as soon as a key with the required key length is discovered + for it.Iterator.Next() { + if len(it.Iterator.Key()) == it.requiredKeyLength { + return true + } + } + + // Return false when we exhaust the keys in the underlying iterator. + return false +} diff --git a/core/rawdb/key_length_iterator_test.go b/core/rawdb/key_length_iterator_test.go new file mode 100644 index 0000000000..654efc5b55 --- /dev/null +++ b/core/rawdb/key_length_iterator_test.go @@ -0,0 +1,60 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "encoding/binary" + "testing" +) + +func TestKeyLengthIterator(t *testing.T) { + db := NewMemoryDatabase() + + keyLen := 8 + expectedKeys := make(map[string]struct{}) + for i := 0; i < 100; i++ { + key := make([]byte, keyLen) + binary.BigEndian.PutUint64(key, uint64(i)) + if err := db.Put(key, []byte{0x1}); err != nil { + t.Fatal(err) + } + expectedKeys[string(key)] = struct{}{} + + longerKey := make([]byte, keyLen*2) + binary.BigEndian.PutUint64(longerKey, uint64(i)) + if err := db.Put(longerKey, []byte{0x1}); err != nil { + t.Fatal(err) + } + } + + it := NewKeyLengthIterator(db.NewIterator(nil, nil), keyLen) + for it.Next() { + key := it.Key() + _, exists := expectedKeys[string(key)] + if !exists { + t.Fatalf("Found unexpected key %d", binary.BigEndian.Uint64(key)) + } + delete(expectedKeys, string(key)) + if len(key) != keyLen { + t.Fatalf("Found unexpected key in key length iterator with length %d", len(key)) + } + } + + if len(expectedKeys) != 0 { + t.Fatalf("Expected all keys of length %d to be removed from expected keys during iteration", keyLen) + } +} diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index a95e568f61..2afc496122 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -18,6 +18,7 @@ package rawdb import ( + "bytes" "encoding/binary" "math/big" @@ -27,15 +28,63 @@ import ( // The fields below define the low level database schema prefixing. var ( - // databaseVerisionKey tracks the current database version. - databaseVerisionKey = []byte("DatabaseVersion") - // headHeaderKey tracks the latest know header's hash. + // databaseVersionKey tracks the current database version. + databaseVersionKey = []byte("DatabaseVersion") + + // headHeaderKey tracks the latest known header's hash. headHeaderKey = []byte("LastHeader") - // headBlockKey tracks the latest know full block's hash. + + // headBlockKey tracks the latest known full block's hash. headBlockKey = []byte("LastBlock") - // headFastBlockKey tracks the latest known incomplete block's hash duirng fast sync. + + // headFastBlockKey tracks the latest known incomplete block's hash during fast sync. headFastBlockKey = []byte("LastFast") - // Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes). + + // headFinalizedBlockKey tracks the latest known finalized block hash. + headFinalizedBlockKey = []byte("LastFinalized") + + // lastPivotKey tracks the last pivot block used by fast sync (to reenable on sethead). + lastPivotKey = []byte("LastPivot") + + // fastTrieProgressKey tracks the number of trie entries imported during fast sync. + fastTrieProgressKey = []byte("TrieSync") + + // snapshotDisabledKey flags that the snapshot should not be maintained due to initial sync. + snapshotDisabledKey = []byte("SnapshotDisabled") + + // SnapshotRootKey tracks the hash of the last snapshot. + SnapshotRootKey = []byte("SnapshotRoot") + + // snapshotJournalKey tracks the in-memory diff layers across restarts. + snapshotJournalKey = []byte("SnapshotJournal") + + // snapshotGeneratorKey tracks the snapshot generation marker across restarts. + snapshotGeneratorKey = []byte("SnapshotGenerator") + + // snapshotRecoveryKey tracks the snapshot recovery marker across restarts. + snapshotRecoveryKey = []byte("SnapshotRecovery") + + // snapshotSyncStatusKey tracks the snapshot sync status across restarts. + snapshotSyncStatusKey = []byte("SnapshotSyncStatus") + + // skeletonSyncStatusKey tracks the skeleton sync status across restarts. + skeletonSyncStatusKey = []byte("SkeletonSyncStatus") + + // txIndexTailKey tracks the oldest block whose transactions have been indexed. + txIndexTailKey = []byte("TransactionIndexTail") + + // fastTxLookupLimitKey tracks the transaction lookup limit during fast sync. + fastTxLookupLimitKey = []byte("FastTransactionLookupLimit") + + // badBlockKey tracks the list of bad blocks seen by local + badBlockKey = []byte("InvalidBlock") + + // uncleanShutdownKey tracks the list of local crashes + uncleanShutdownKey = []byte("unclean-shutdown") // config prefix for the db + + // transitionStatusKey tracks the eth2 transition status. + transitionStatusKey = []byte("eth2-transition") + headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header headerTDSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -> td headerHashSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + headerHashSuffix -> hash @@ -74,8 +123,94 @@ var ( currentRewardGivenOutPrefix = []byte("blk-rwd-") // key of SnapdbInfo snapdbInfoKey = []byte("SnapdbInfo") + + // Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes). + SnapshotAccountPrefix = []byte("a") // SnapshotAccountPrefix + account hash -> account trie value + SnapshotStoragePrefix = []byte("o") // SnapshotStoragePrefix + account hash + storage hash -> storage trie value + CodePrefix = []byte("c") // CodePrefix + code hash -> account code -> We not using this at the moment + skeletonHeaderPrefix = []byte("S") // skeletonHeaderPrefix + num (uint64 big endian) -> header + + // Path-based trie node scheme. + trieNodeAccountPrefix = []byte("A") // trieNodeAccountPrefix + hexPath -> trie node + trieNodeStoragePrefix = []byte("O") // trieNodeStoragePrefix + accountHash + hexPath -> trie node + + PreimagePrefix = []byte("secure-key-") // PreimagePrefix + hash -> preimage + genesisPrefix = []byte("ethereum-genesis-") // genesis state prefix for the db + + ChtPrefix = []byte("chtRootV2-") // ChtPrefix + chtNum (uint64 big endian) -> trie root hash + ChtTablePrefix = []byte("cht-") + ChtIndexTablePrefix = []byte("chtIndexV2-") + + BloomTriePrefix = []byte("bltRoot-") // BloomTriePrefix + bloomTrieNum (uint64 big endian) -> trie root hash + BloomTrieTablePrefix = []byte("blt-") + BloomTrieIndexPrefix = []byte("bltIndex-") + + CliqueSnapshotPrefix = []byte("clique-") ) +// LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary +// fields. +type LegacyTxLookupEntry struct { + BlockHash common.Hash + BlockIndex uint64 + Index uint64 +} + +// headerKeyPrefix = headerPrefix + num (uint64 big endian) +func headerKeyPrefix(number uint64) []byte { + return append(headerPrefix, encodeBlockNumber(number)...) +} + +// accountSnapshotKey = SnapshotAccountPrefix + hash +func accountSnapshotKey(hash common.Hash) []byte { + return append(SnapshotAccountPrefix, hash.Bytes()...) +} + +// storageSnapshotKey = SnapshotStoragePrefix + account hash + storage hash +func storageSnapshotKey(accountHash, storageHash common.Hash) []byte { + return append(append(SnapshotStoragePrefix, accountHash.Bytes()...), storageHash.Bytes()...) +} + +// storageSnapshotsKey = SnapshotStoragePrefix + account hash + storage hash +func storageSnapshotsKey(accountHash common.Hash) []byte { + return append(SnapshotStoragePrefix, accountHash.Bytes()...) +} + +// skeletonHeaderKey = skeletonHeaderPrefix + num (uint64 big endian) +func skeletonHeaderKey(number uint64) []byte { + return append(skeletonHeaderPrefix, encodeBlockNumber(number)...) +} + +// codeKey = CodePrefix + hash +func codeKey(hash common.Hash) []byte { + // We don't use any prefix for code key, otherwise we should return append(CodePrefix, hash.Bytes()...) + return hash.Bytes() +} + +// IsCodeKey reports whether the given byte slice is the key of contract code, +// if so return the raw code hash as well. +func IsCodeKey(key []byte) (bool, []byte) { + if bytes.HasPrefix(key, CodePrefix) && len(key) == common.HashLength+len(CodePrefix) { + return true, key[len(CodePrefix):] + } + return false, nil +} + +// genesisStateSpecKey = genesisPrefix + hash +func genesisStateSpecKey(hash common.Hash) []byte { + return append(genesisPrefix, hash.Bytes()...) +} + +// accountTrieNodeKey = trieNodeAccountPrefix + nodePath. +func accountTrieNodeKey(path []byte) []byte { + return append(trieNodeAccountPrefix, path...) +} + +// storageTrieNodeKey = trieNodeStoragePrefix + accountHash + nodePath. +func storageTrieNodeKey(accountHash common.Hash, path []byte) []byte { + return append(append(trieNodeStoragePrefix, accountHash.Bytes()...), path...) +} + // TxLookupEntry is a positional metadata to help looking up the data content of // a transaction or receipt given only its hash. type TxLookupEntry struct { diff --git a/core/rawdb/table.go b/core/rawdb/table.go new file mode 100644 index 0000000000..35cd11d55d --- /dev/null +++ b/core/rawdb/table.go @@ -0,0 +1,313 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "github.com/ethereum/go-ethereum/ethdb" +) + +// table is a wrapper around a database that prefixes each key access with a pre- +// configured string. +type table struct { + db ethdb.Database + prefix string +} + +// NewTable returns a database object that prefixes all keys with a given string. +func NewTable(db ethdb.Database, prefix string) ethdb.Database { + return &table{ + db: db, + prefix: prefix, + } +} + +// Close is a noop to implement the Database interface. +func (t *table) Close() error { + return nil +} + +// Has retrieves if a prefixed version of a key is present in the database. +func (t *table) Has(key []byte) (bool, error) { + return t.db.Has(append([]byte(t.prefix), key...)) +} + +// Get retrieves the given prefixed key if it's present in the database. +func (t *table) Get(key []byte) ([]byte, error) { + return t.db.Get(append([]byte(t.prefix), key...)) +} + +// HasAncient is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) HasAncient(kind string, number uint64) (bool, error) { + return t.db.HasAncient(kind, number) +} + +// Ancient is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) Ancient(kind string, number uint64) ([]byte, error) { + return t.db.Ancient(kind, number) +} + +// AncientRange is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) { + return t.db.AncientRange(kind, start, count, maxBytes) +} + +// Ancients is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) Ancients() (uint64, error) { + return t.db.Ancients() +} + +// Tail is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) Tail() (uint64, error) { + return t.db.Tail() +} + +// AncientSize is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) AncientSize(kind string) (uint64, error) { + return t.db.AncientSize(kind) +} + +// ModifyAncients runs an ancient write operation on the underlying database. +func (t *table) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (int64, error) { + return t.db.ModifyAncients(fn) +} + +func (t *table) ReadAncients(fn func(reader ethdb.AncientReaderOp) error) (err error) { + return t.db.ReadAncients(fn) +} + +// TruncateHead is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) TruncateHead(items uint64) error { + return t.db.TruncateHead(items) +} + +// TruncateTail is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) TruncateTail(items uint64) error { + return t.db.TruncateTail(items) +} + +// Sync is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) Sync() error { + return t.db.Sync() +} + +// MigrateTable processes the entries in a given table in sequence +// converting them to a new format if they're of an old format. +func (t *table) MigrateTable(kind string, convert convertLegacyFn) error { + return t.db.MigrateTable(kind, convert) +} + +// AncientDatadir returns the ancient datadir of the underlying database. +func (t *table) AncientDatadir() (string, error) { + return t.db.AncientDatadir() +} + +// Put inserts the given value into the database at a prefixed version of the +// provided key. +func (t *table) Put(key []byte, value []byte) error { + return t.db.Put(append([]byte(t.prefix), key...), value) +} + +// Delete removes the given prefixed key from the database. +func (t *table) Delete(key []byte) error { + return t.db.Delete(append([]byte(t.prefix), key...)) +} + +// NewIterator creates a binary-alphabetical iterator over a subset +// of database content with a particular key prefix, starting at a particular +// initial key (or after, if it does not exist). +func (t *table) NewIterator(prefix []byte, start []byte) ethdb.Iterator { + innerPrefix := append([]byte(t.prefix), prefix...) + iter := t.db.NewIterator(innerPrefix, start) + return &tableIterator{ + iter: iter, + prefix: t.prefix, + } +} + +// NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset +// of database content with a particular key prefix. +func (t *table) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator { + return t.NewIterator(prefix, nil) +} + +// Stat returns a particular internal stat of the database. +func (t *table) Stat(property string) (string, error) { + return t.db.Stat(property) +} + +// Compact flattens the underlying data store for the given key range. In essence, +// deleted and overwritten versions are discarded, and the data is rearranged to +// reduce the cost of operations needed to access them. +// +// A nil start is treated as a key before all keys in the data store; a nil limit +// is treated as a key after all keys in the data store. If both is nil then it +// will compact entire data store. +func (t *table) Compact(start []byte, limit []byte) error { + // If no start was specified, use the table prefix as the first value + if start == nil { + start = []byte(t.prefix) + } else { + start = append([]byte(t.prefix), start...) + } + // If no limit was specified, use the first element not matching the prefix + // as the limit + if limit == nil { + limit = []byte(t.prefix) + for i := len(limit) - 1; i >= 0; i-- { + // Bump the current character, stopping if it doesn't overflow + limit[i]++ + if limit[i] > 0 { + break + } + // Character overflown, proceed to the next or nil if the last + if i == 0 { + limit = nil + } + } + } else { + limit = append([]byte(t.prefix), limit...) + } + // Range correctly calculated based on table prefix, delegate down + return t.db.Compact(start, limit) +} + +// NewBatch creates a write-only database that buffers changes to its host db +// until a final write is called, each operation prefixing all keys with the +// pre-configured string. +func (t *table) NewBatch() ethdb.Batch { + return &tableBatch{t.db.NewBatch(), t.prefix} +} + +// NewBatchWithSize creates a write-only database batch with pre-allocated buffer. +func (t *table) NewBatchWithSize(size int) ethdb.Batch { + return &tableBatch{t.db.NewBatchWithSize(size), t.prefix} +} + +// NewSnapshot creates a database snapshot based on the current state. +// The created snapshot will not be affected by all following mutations +// happened on the database. +func (t *table) NewSnapshot() (ethdb.Snapshot, error) { + return t.db.NewSnapshot() +} + +// tableBatch is a wrapper around a database batch that prefixes each key access +// with a pre-configured string. +type tableBatch struct { + batch ethdb.Batch + prefix string +} + +// Put inserts the given value into the batch for later committing. +func (b *tableBatch) Put(key, value []byte) error { + return b.batch.Put(append([]byte(b.prefix), key...), value) +} + +// Delete inserts the a key removal into the batch for later committing. +func (b *tableBatch) Delete(key []byte) error { + return b.batch.Delete(append([]byte(b.prefix), key...)) +} + +// ValueSize retrieves the amount of data queued up for writing. +func (b *tableBatch) ValueSize() int { + return b.batch.ValueSize() +} + +// Write flushes any accumulated data to disk. +func (b *tableBatch) Write() error { + return b.batch.Write() +} + +// Reset resets the batch for reuse. +func (b *tableBatch) Reset() { + b.batch.Reset() +} + +// tableReplayer is a wrapper around a batch replayer which truncates +// the added prefix. +type tableReplayer struct { + w ethdb.KeyValueWriter + prefix string +} + +// Put implements the interface KeyValueWriter. +func (r *tableReplayer) Put(key []byte, value []byte) error { + trimmed := key[len(r.prefix):] + return r.w.Put(trimmed, value) +} + +// Delete implements the interface KeyValueWriter. +func (r *tableReplayer) Delete(key []byte) error { + trimmed := key[len(r.prefix):] + return r.w.Delete(trimmed) +} + +// Replay replays the batch contents. +func (b *tableBatch) Replay(w ethdb.KeyValueWriter) error { + return b.batch.Replay(&tableReplayer{w: w, prefix: b.prefix}) +} + +// tableIterator is a wrapper around a database iterator that prefixes each key access +// with a pre-configured string. +type tableIterator struct { + iter ethdb.Iterator + prefix string +} + +// Next moves the iterator to the next key/value pair. It returns whether the +// iterator is exhausted. +func (iter *tableIterator) Next() bool { + return iter.iter.Next() +} + +// Error returns any accumulated error. Exhausting all the key/value pairs +// is not considered to be an error. +func (iter *tableIterator) Error() error { + return iter.iter.Error() +} + +// Key returns the key of the current key/value pair, or nil if done. The caller +// should not modify the contents of the returned slice, and its contents may +// change on the next call to Next. +func (iter *tableIterator) Key() []byte { + key := iter.iter.Key() + if key == nil { + return nil + } + return key[len(iter.prefix):] +} + +// Value returns the value of the current key/value pair, or nil if done. The +// caller should not modify the contents of the returned slice, and its contents +// may change on the next call to Next. +func (iter *tableIterator) Value() []byte { + return iter.iter.Value() +} + +// Release releases associated resources. Release should always succeed and can +// be called multiple times without causing error. +func (iter *tableIterator) Release() { + iter.iter.Release() +} diff --git a/core/rawdb/table_test.go b/core/rawdb/table_test.go new file mode 100644 index 0000000000..aa6adf3e72 --- /dev/null +++ b/core/rawdb/table_test.go @@ -0,0 +1,128 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "bytes" + "testing" + + "github.com/ethereum/go-ethereum/ethdb" +) + +func TestTableDatabase(t *testing.T) { testTableDatabase(t, "prefix") } +func TestEmptyPrefixTableDatabase(t *testing.T) { testTableDatabase(t, "") } + +type testReplayer struct { + puts [][]byte + dels [][]byte +} + +func (r *testReplayer) Put(key []byte, value []byte) error { + r.puts = append(r.puts, key) + return nil +} + +func (r *testReplayer) Delete(key []byte) error { + r.dels = append(r.dels, key) + return nil +} + +func testTableDatabase(t *testing.T, prefix string) { + db := NewTable(NewMemoryDatabase(), prefix) + + var entries = []struct { + key []byte + value []byte + }{ + {[]byte{0x01, 0x02}, []byte{0x0a, 0x0b}}, + {[]byte{0x03, 0x04}, []byte{0x0c, 0x0d}}, + {[]byte{0x05, 0x06}, []byte{0x0e, 0x0f}}, + + {[]byte{0xff, 0xff, 0x01}, []byte{0x1a, 0x1b}}, + {[]byte{0xff, 0xff, 0x02}, []byte{0x1c, 0x1d}}, + {[]byte{0xff, 0xff, 0x03}, []byte{0x1e, 0x1f}}, + } + + // Test Put/Get operation + for _, entry := range entries { + db.Put(entry.key, entry.value) + } + for _, entry := range entries { + got, err := db.Get(entry.key) + if err != nil { + t.Fatalf("Failed to get value: %v", err) + } + if !bytes.Equal(got, entry.value) { + t.Fatalf("Value mismatch: want=%v, got=%v", entry.value, got) + } + } + + // Test batch operation + db = NewTable(NewMemoryDatabase(), prefix) + batch := db.NewBatch() + for _, entry := range entries { + batch.Put(entry.key, entry.value) + } + batch.Write() + for _, entry := range entries { + got, err := db.Get(entry.key) + if err != nil { + t.Fatalf("Failed to get value: %v", err) + } + if !bytes.Equal(got, entry.value) { + t.Fatalf("Value mismatch: want=%v, got=%v", entry.value, got) + } + } + + // Test batch replayer + r := &testReplayer{} + batch.Replay(r) + for index, entry := range entries { + got := r.puts[index] + if !bytes.Equal(got, entry.key) { + t.Fatalf("Key mismatch: want=%v, got=%v", entry.key, got) + } + } + + check := func(iter ethdb.Iterator, expCount, index int) { + count := 0 + for iter.Next() { + key, value := iter.Key(), iter.Value() + if !bytes.Equal(key, entries[index].key) { + t.Fatalf("Key mismatch: want=%v, got=%v", entries[index].key, key) + } + if !bytes.Equal(value, entries[index].value) { + t.Fatalf("Value mismatch: want=%v, got=%v", entries[index].value, value) + } + index += 1 + count++ + } + if count != expCount { + t.Fatalf("Wrong number of elems, exp %d got %d", expCount, count) + } + iter.Release() + } + // Test iterators + check(db.NewIterator(nil, nil), 6, 0) + // Test iterators with prefix + check(db.NewIterator([]byte{0xff, 0xff}, nil), 3, 3) + // Test iterators with start point + check(db.NewIterator(nil, []byte{0xff, 0xff, 0x02}), 2, 4) + // Test iterators with prefix and start point + check(db.NewIterator([]byte{0xee}, nil), 0, 0) + check(db.NewIterator(nil, []byte{0x00}), 6, 0) +} diff --git a/core/rawdb/testdata/stored_receipts.bin b/core/rawdb/testdata/stored_receipts.bin new file mode 100644 index 0000000000000000000000000000000000000000..8204fae09bdef1db96cc7ababca9b938910f8a75 GIT binary patch literal 99991 zcmdRX2|QHY|NqQbv&$M0*>^3Hq(r17$cCw^swPZ^}D6(aVRQ7}xOJs{I zks>>pZ2vps!M(a;p6Qw2^L+o~^>RP=oXdIQL?9MkC9EbI}|`LWLTVwATKWIKgZ~6!6tr2vXRSd6$EY1Q81#zZgoQj^?if zA(=xjT0tN}V?BiBjAj!nd6K{0@m5LnId$v=L}5{R(JL@Wqae5oCsfdvk%`+EFZo@9 zOB;gj8S?jwpdozF)8EcwmtugcUcX(13s}q|I3BK31;@k4NP@Y5xoAFUq6MB^Z}hF3 zS^YQJOJ773Tdg?fd4GLu?GvOQi<)ZL70R~u+6|Y`^GSe?5OyI7aJ)bi;4+S{JBFtY zgag5|C0E$t)kh}$u1(Yc@2{lsXAjOpA=z65w+J3TIHCr4qy)DJlSKX4-z5e=wTqAt zi^Yn!gC}a7L6A8(+Zd+nNH6*~TQN*N07p%Q?!niky98qRj#GxI-7Qq|;(bl*AXp^A zIprQNs}a{ko$ZQ3$$jtjozI}brbPsTceg0Fe`GZDNGn-oC2-zwO$|f=tPIHUuD0P= z*N3Z9#DZhaVH?O9^IWy(>n+Nfee|B14tK{Pwuu3Ux;V4x3npOLgjCECB8DLPCCrJ8 z3HqC0kgJ2>E(EA6;d8|7IkCUKTfkV1W49ni^sRUqXRp~z17aVu@rw!z;+TP>E(g!l zYj)ZXS=9;pgS76V0WVUBJ8)&NQ*}Qi9orQ~2`{?tH1@$=w36c;Xj= z@w+ThJuO`3&vEgaI?uJ4C+`AHSHMF zR6gf3W{O>u3f~Nv)&^c&IpK|_eY^MWIq6-EL1)%RH3yG9UBBIg1@Xm-{_Co+?OWxU z#n4OAvrUP%S2%9w?8bR-9nS^#?7*c8OY{*4bf@7>6%H|DZ8ozj`un@d9vqVty10GR z_nFvs>CerNX?vd`3&0n_Q9w*QNSFNCgRL6hsy#qKzlk`4ehVdO0HM({(u9CH!BKnD zG_Vd_w+oIZQvNM{xG~3aX_BZ-1WYXNUvR#G#sh4j@bOnoqDJU{$M*8*mjK0i9*LME z2##*k!K=;BgJSMq;rpbnulMRo&)sHx%pvZzeY=;GdaeQh_nd)az_o@clSTW$oI1B<)71G5!E z$}M_HPZL2?_RJ68f=gV zuRylcCfB5#i+wQjZ`CJ)=s`JIk#Jgb^0tMw8FttYcOl?7|J|^EX05~lDV+dABNPZS zSX_V)4xE<-J!z)PHas5G>QbAU4}%`pWTJ+jj<#@33eAu3;g1$IByuc*enW6JyqE#x z)JgsowqtKjPfcKgn3Ph)z3|OU5TwE>nc2+x4xE=kmcFc0Mpv`@iBC?JvcaIF2 zthqixcI1xE&+x}aH&=8_hd{vD2FDZXdEwe)@b!e!2kQbdZXUlR?61?stYc6! zNH1opy%^LuiHgX8;~KrOEO)BSNc%h8M&FI{8x1jNFpN0oD&)p=A6oI%`o45gQPGI$ zkChMwkg{%fg-iRD54hWza04S^yDPXF3b9QPJqR2!{dMU|{zFU?Bw~&rF$B?rz%4LG zs)niSj*1ga-#4kwAN^}M+t^$veA&zhQkMhKfP)9$V9d^FZC3cv`fz|6bj^Y&cur3` zRY5p>L(PuKg+Pe#w;QG`3?;B2$Ertxp(NFU9Ngj$8jNmrqwq+(jI)jChRLrYPOp>> zggfqoD1em#nd~C}NF#|lzi5OlLg#@Y0)a7ogf$cOs0KP#B#0giZd1vRQBN+ujQ}{r zqwLLJbY~k5J^V4K_;!YGhYUj+v9*&RF_9ZbR{{Ktbyvb_Bh%JmGV_yMngOZbny0_D}$zhw_K z>nbTJusM*tp~PhUP7wtu>@E*D(zTa=ZPoP`Z!$f&|pMxH4krp zgqUy2@bpL6;3>aWIuWpi1(}@EN!-{|MvhGX{6g3g;cu>w>te{{kzQ;a+88n!5^Y4; z@bz16@wSe0^<_MrQx|1QbK&7Iym>DaKm_Z5QO<&Hoh?M}ZPH zIvB2K=Ue4@QuFVC3uD~4O8vof7FCMdf$B>-IcF`AF2r^>1Z^AB_VaH$TKqG+Li`@B z&rZv`8mJO{;85WVxgCZ_f|-DZ1uo$~Zdee=_O)aeFZ@h9>$^}&P+{981J7L7lKOt2 zVSzJEY{NpZ(BOE%*RVjL!kx{#99$d_wm>YN_=Vt7q!;_?kOJ(XAPRfnSQ%e{VL&3i zcv<-|^cWIVCxTPZ1wep+dD{^E3K+wZs5%j}o$lI}E1sYKneC)g;C*uL1I*XPa?xNw zV0rDt$9Uuv020X)WRib7>B>yk!vN8rPcI@TD<4L3_5rsLg6K5mb`Q{`Vh&M#5*e5B zt6hmA%MtUYwPf6K-s;usjbR95yG@+Dqo5!x?U#zQ*%XB<1(`xe&5vudl9Ed98*o9v6-} zlSZnF2V1Ye#=o4bK;7bq)VB5ClhmF-B{jXW=_C4|^mxcVDJsABU=uL12}~F`=-;Y% ze?|`&DO8@zJc;gc!4(bITtCbY91k~;2FDXhAFMk+xdMkl<-f8X3rqpal_u`tZ+wW_ z$hG>^jGFo0j1rEYKRLn@p4d!3T$POm8-}}?h;L)&!g>=hVO_(mhMl*rvDGS=05J3M zpTc2=yKhMBGf(IkM<^eyk z;H;fWY**AIkG{_n3+}}< z7?Rr4VK~O_art}c6g25IRWj3bfO+#M~ zhtKR=41-EcMVTW3#h-{?w|F?yD#?J|;g`qQDITQ{C|H=x0Y z;MjpEfR(qf?U+JbSjU39@IQ4d98es*B}Kh(`s#Zg*{omd z)g@U1PPad&b5UAVH`a3Lc_oka@9}dXHzo$`buDU_A4{OdKH^`jB z7bGy!j_ZO1AJLGk3sSc`IyO|vqBZbh^I=)P)F9O(yX(~+jxhB?AYD2Acyvd4zo8+8-t>O)8R#K?b7|jtHa0Fn&@n#I1GWLFql*u zB;P3apsYEXvk}!z>!7kHfsn=9ut9!x~hA z<@V`~a*=d}hHbON^34B$w>|Y0Pf*do+eE%Q$xNc6`nw^J-i}O`lbyR)klbH8{q2?f zb7tPJKOhQAG3yBIvH^+SP2g zr-CRfphR-LEdFk67|ITer2v$YLPtr)Wk|?al$L#me4h7gtR>^g|gd|R}h7p+1*#L zQaNoic*z@rhhUsLp!8_J;Cxp0$8D0KBv}8SZ%xrHexcVL&tM9TQtkw zC-0%0Tn;62!bo=qK&hP1G>+0u3@>nCTQXDx?{@RD=qY}u42`6^%6vip6|tQa!Savo zXmjQLA>rG9#gVzvlbxpB#QHoQuyjZ2;@gyt0kK$0H`Q;(q1>#N+thl)c9uMp&ys!( z>q`Z)xGG{%5-j*Nj4#(nGbbS!`>4)N=I@N}tx1JWGZ70v@q*-6>4agtOPwVtKY%Mk%&y0sl99&Ae4>8#0X)~W3uHY3oR+0=X!R85b z+UgS{+Tj(cAhLWIrt_T|xdh!axpMoi#@ZjT8u_mt93YL*e{{*j6{Xq0!!ePR4U*>J^?W>cO-pG~4qb zAY8`-8s4)f?Rsg!+t*PjM*ge*_!L9|vG7AasbFXhQ7xNmbyYat{Cg}#a&4&F}_)wlMUtO@Cb>c6SeRIvN9Hol(?dc8FJ5P?H>KffCCOR`yNzUbw z*Y9SPHXMlPbDI*_T2fX%>|Yg*2PI;YTzT@VRv;Ez;;L0*I4&bXcB!(jPu-b^YN-*t z_#>!#Pf{YuQGR4lPItSQN%nXu^Y$(CZ>LPCDkbruWIpu8^OoCCBk(BmtI-MRvb#TeP3}XNpWAQE3zh{GjFG9 zeI&JK&E_{fu4ad!AYvTtdGApvQp6`m39i^rmvL0bVk2IZ0F4Niu8~n}iK}5UaVSaA z8{F28hyEn<;J17kfV$POuZFVDoC+?*ZG$C((zOI5-j> zhQ{4*a)zV;$~BVhhWZMYO56)vVbK%$>RW>7{|p@93mRXeNanEu;WpiA+0`GNP7A45 zW%++<%+ptfFhsP+(voEqU_EyC2f5|iD5CEF^7OtMYIJ4xbGe_FAO5^;t)qnvLQYx>+)=Od5S+TXBN6XLzR4)6`l{SqfcmIv2sX-(TedzSTvP zo_69zk6UfPNX;<_F5!Q$F@p^`>=?QcoLPzeLxON!xV(^yh&O=VZ)kHI_T#(f|56m;>& zpuynxux_;T%3(ObUa?M&MvC7~yx5!`q5xLL%8KC?*r$E)fFr6!CuqB|{XNI9*&4wV zuMjS>uJhAxV=fnTg z`ZNy$-;qQ+b>Nss$bu%LU+DNeQ z7D<{FXZFX{IyU5MqaE{|;% zMzjNT8}<5s>8l;Bprb=h$%-@2B#XO4AqanBs<(Zyqz!}X4(;3jzbJgLiNi>Tqi zhQrMdJd6%iYgE+Ptaq6_##blz1(Gee8bWwD0FbYrPq!ROqQ(U#Sv9@q-C1hHgc0bF z_|lc3(GFq|l$wz@lrwg#Rk}Pb-}hEc+f(#%vKB-E0sV0g-blleZ@|ooW!B)tYTO}9 z5PLXCC5by6-iV|7(0(40Wkh`=EuR;a>A(lk^$)eqTBYCby?Y3P({3-b=EY?5sK+ig z&M7973d%QC-$4|h`9L7gyg| z5QQb#LavlPjlfo6BkQ>v0c%2)A00h1me0d+93%8H`I8C~<4x?utv>TrF4~;4OfqK! zr~AarW~#(I4+scxy@)-SK83V(DowVA~n^x?U;iX0ZbQ`_o!t#8S z5x1l1s3~qo)6tCe*uY~HTX2ViDg_vnD$>goJ&8dnPDBRz47N!=yOX%){w`)2Ph*{^ zp|~h1V4;qv+RGL^Gj^5#1xu&n^i7ejcW&8V3YNc7jLiVv9AP6BBYp^Z;73PPttE&q zf8UX7n{yb7?WB}tB;U1Lw{!O|-~G>_n+|gBp*piKHa3}>%1DU%i|5C3^=b)#yAq~Q z)wb+9)-MHt%$mO2Z@69$R|}ArtmB?JV13AgvK*3g(xb!jmt%tczufOpvei%RmwjAf zwFMU^-xse@WX$3B-<%yEVJoYDIf*w3-?b>#o{a(8bGLMkAEd(7-CRezA^Q6i>wv#lg-90{G zN3s@%M?`_cP)V@!UW-6UkyJ}tJYSU^1m$C+lNZFJJS{>Te1_C0)=!4i)o4Q$z*z!h zA?Nd=%2<>q1`d-9rNf|FN<)u{aNUsrXK^=bv1wWs=*I1j_tzGhJm@mAP*(f2&*g3V zji_hK?~C4x$X0oPiahwma8%IcLLZ!5AUjbtZcwx3y9$E6z4gx%k?_=~Fy zPAd$r&I^?6sdh0moPIBXfV#w@>YQYEq*@thkIW)yUR=gLY+&%wSf zhYbPB4A1XgC%f$jd>Lsx!ExsZFWTgDE5*q#>@HkPm-Ttgo-({)tqV)25~$U|YX(aO z0zqH-jBHMV`)|3gI6VRyVfVt++?UP%$mP1PHwP)Lr=C2!>Ic>M?F{`}2qXYi06IPS z;mjSn{`VG&eIuCWW3PJ=NRK#jP&}N-1Q&=4vl#VB&_vqcjkl!R#uIA14u8D#{Vh#- zc%u=P|1^F>ng1rdJre?9xv`u7o2)2a00iTIFF#0HmM2*rCAhRe!oVj4tPpkd%M%ZjXB0t(BNPi<`WGa2v|P}$1)eJo`fH0<{?CR!@g3JN4810y{( zf}D(wW~mCWWCdHhPN0!1rq1YBDqG%1O(xc-x_^D1bWMrh*Mw%ubsJMfDe5eGX;RR0 z`bq){1V)Xf@>-+#!BHIc)BX@^(ky!}d+MsqYe(Oq)AE;@T5H$&au|-QPeDf;5MY1R zYXaa1pvQp&+Iwr%TuVxd_WpI@c$eCtcT(aL%=%vmRU<^7rO3a z(8w2VB}z3eG9NA9-36)-ooyi5z+nUZqmK2(^-4B__y0YVO`cA{_Ah&4-bp%acv;eJ z!Zcbmvn)!nXCk=ai0xO>4L}KUsUt_JAFGDEs(3lr@Tp)+@s4eX<*S{_u2RHA9MbE| zI>U)5V9ND5JQ^tzd#|wO0cB_Ru^~rS3FGI$q=u*|7pSNfLN7q{8@XQi zsMr1Tm*hB>f2II|lv?S5g=y!k{yz;~Mb{oGH`TCyj*Vms=k6-wY zOtc~oL)d(L{*kNJsVa$v6JeRY9y30uB%a$4g@r;tP8Lx?S=P}4CPtZ=$u{O z^o~h;`oz)6LZ6R2K4PKFGOh5j=7#e9J7Zuhr}?4;9pa6+*{EC zd+(bR#Y)gNB_Z@678i;Y`&>WVg@Dr$_3|>d$GL$BR9}*#O6bbDW;#N(_#vyb1kEMe$XAhyS2G!YPDdF)kx zKg8!rao-LGmv|Zz2$sU3TOlmDwmTwexmrJXptwh`_R~TX02aZ&&0NMzJXlJdLU-d( zzLYzqkUw0}bmNYOu2h2~t37|E1FoO$p=qV4ue_AQMkwtxJXiw0X=$CUT( zj7jRPkN#`c@Z74Oh#E3fhA4oPsret%?_>>~TeNnzDZ`;$Yx3(=m!9~=EwS1!WU$gDr2>~Hu+4n?jwdH4;VMWUGc6_(HZc1c9 z6hJ6(BE?uc0?Va~EkDWJQ;;t~?lrZ1u`|1hsRLiF=&s@C4G<93KW0QPwby8TIi|Nm zWK;OXz^XQHh{95oTs7xCa45&#<4}@fRZt`Fz1G#y564eOerujS8}otf;dyA&{K~BV zKL3wvQa9Wj3VhDfAKPo3nuiZ1x$)bJE~nLS?UL)RkGM+YrollnO8xm-*NIxMUTdIv z5EbOPGm_Rz7rKt>@5RuQAQWVJDZ_}2z9~L+96{5J4=j#dY{3IH-YtAh0rt=*f<3rz zlbO3+cljmR-0eE;E3$1F66AS_2M>0oJV?RVbvnuE(Qtu_VMuB0?K(W@h?+YRM5o{Wu_H$=zH}F2tJPzEvIl2VE-c%a z0wA&INOQaH{{r3Lxn1{)KVZNR$tac0aGFoKNsp}d&>Ed}yoAw5>^rr`KPEG9O&r(Z|=+O9)`O%r@05?C24Yew|B zPjn!?d{(7TG*2`Jxovs*t+9tOrW!4Iwe9;iZh@_>a_#6~p;+A*wn3TIM`pf0_7yEu z{v1#Fj!d1RQ`2w%@b~~<4ti<2(DN$?!RO-Y!?TT1s~!b8bd0aHYkOdrPh|j(|6SgX ze|7m(P`$P)^a|D$d5v19(p~rlR24KdHVt0?*eir>H+?z`U8l-G(>!rbL^~W#!@U)`Weu(4`dEUF>-7mo5ZjQTjHKZtDy^ z6;+7Bf#J!xdsSy7RF1q?&b$MiErxQhK1dfst5|G*-H18;gbDJ^<`guPP3YO-;4xgY ze&yBNlTmmEazt^-_*G)VBhPP{Uug4I@6Hu`>p`Ud zOIelut!+qtf%>=-FCf8-yO`uZeJFY~nwO||!c%FBdQ)&>DCrpA01z)bWx0SG< z^rQ+I{B_BO38H{l{2`45S4OWdUiWCC8|ihmPG;isL`zVq>ko$4{@zg>7rCXcxydSO zCEL~ly-abPr{$}Zb9c}2P%0VbE16%?Q#q$PTHqzf=#-Qw#dsWIbef{5nPk1i*0SNgqY?HwyY4l7`>Js%Ri(b1SB`$oMGk5hs3Ng$ULOi%_GJbCU&j z-U~KGeAW)wg0F{>@nlUf=G);a+4UX8RoEW!7K01P-0a$;Edw5kH#58{x33W9&W?Ar zzY1M1KVCVOyxJr*QiFQ+om5uC_Mufp*U&J?=Yvo;<5RFxh+8J!1sJ3oAPmxDkPZ{i zg5Wk3wxds9t>d7u6J14N(!xhKp6@8C;>)62%^`34`N8q_!VQDaXCrDKYf-3$?KF6M zSo@uk@v5g)yV6h0TV8rZ-atJ;$+RsEUV3h_G+^mD03FV3JK8FPO`|m-BbzGo^T@#u z!kynX`U`%Ir>5b*cDl8{0)Lm+7{+v|MfpWO5 z!iJs0-45a@n7EAex+5+!;S4-xV{HEO=?V*Pjon&b$}aG~8-G9aZGAgBc9p4N*>r4~ z!wHXL9S;sW*`6(|b-2wFdS^tr(foz;5VYTuAK=dtH0x<^e!nZ`=@{jT5_^lj`5Y`c zl1(3F=MjplLSX&(PU}vR`R~~o!K276U9N!|v*fA)T?5u?>$=cjsPEJY76+n^QwX9Oq`c9Rr+>i3I0@>$B9@the#L*L5`$FDa1s+PSMS6w$3}eXN0wcb3kq8T`%3Lc}_Cd-O zPmb>_sMO>mcEsoT=}Mc295>sc`xkpRYToHS{-{J44F;6%f|S_w^Q zD&24W=R7C8u^$S9Vf>GuUev$K8#(N;KPk-l9n|2$8O?fhE%%!1^nBP4g~6BxJc_hW zEyM8w?dBG+Ar@U~F^sg*0`X4pUZH{}>Q&xwnM314ykHyX(85HtN!Fq9ZhO{WDjl#d zQTt2>%kdm?CuQk+2#Dz?Rv}&-x^)}YOt2J%2fnu4h)#wmEJa)v?M;P2dLg|c;xQPc zGyeOd?Xy!Qca+Gp4v*G^`%l?4k@p(XZ&jz#DN>d+GAd_^gjnGcUmY0G*{ATd_AJZm zV8g_FwR*!41uzVh;rJ@+YGD0uItcDMpfk1i_S271=gOzAQMFf^Dreu;CmqMHXH|_X zXe=K%6hx0VX`C6h`Ywoq$N6LXlu#uu?`zLhtZo*2!~QG27NP*xh~$_cK)>?vQ=!Ld zDhda7m{Ekhio=YyM~#7^;Y3!rKy27T_5ojsJo^jhD|^;5n1rMsOV05+^kygJwe=c8 z>KBOZ0QPSdy=%Kxhb9??OmZu`9LUcj(1Zqq;ONWHDqVT0`LN<_%7s0zk5Uh8y9rSM zDU+Q-g}JXV$32D@S%SiLTld&$a0g4=+u!fy;vVdYqQgDd6Lm}+H|M>t&q6Za&uTEr zcgg6dH^_L-=W1M%BcnRT41voWPj(?Xtz4~C-H+!^_~)&%WTi(T3ZVR8MWk_JGj7Ov zzeVHUW8L@GzV&JKkJ=w=m7pb3+%en2r8eM0NzUXi_PvAcEe<8RcHO_nWn%}IAnSqd zH=%JVyT3^!8qunzx8XyHo3pvDKDe-l9c4v%lU@ytH8%fax5%-m)nGO9?ehx^E0#OD&bj{-dOmW= zk$2MWX?ca4Aa>vtJs5^{-58Z<+#x6dcj$hCJv`X_40pIzh&y~L#~xy(t8xAuCnJt4 z13U}4L=CIYo}^Qz$n`09dH?Dt-Oj-7GhZ)!fPmK&*<^Y48C{avU{4SV{oIMy$qfIS zTd=uRRLE1tmO;FX7s*f(Y(qY8@%b`J{vD+GOi>)oPY+~zSi&ETRwT0%uY$Xo6;+! z!BLZX7NP)BCQooRIid*Mkq5v&G>Zi`_Gux#5*ckU6lx^uC5akRkoi>xtbehYRG%^? zXgjmr7HQcfC0FrI($ptY#j{Ve{1ZN^5Fl`*s2T)qmt8COi`Rd~ZPz*~Qg+c_OLdt1 zI$m(tGTq$$FcKJn!zKL3BXDfG2y<}*udgR@;&1;bRaC%b$A%mc(UBeSugI*{~wetR|YC2lkE}MS3Nj_r;*}f%U56 z?^4yxnf>b7$8BCICcrtZ>%Q;J~~iQzt$-yS@uP!E;*v`Imhm_6eUM z3Si|qnW6x@n)W}7uJC^vU6BV+$>6waM>AT4#+f*7(JOv2OAYtUYX7>b=N zt~>Y@EPbGB%5Vkuds#t3$S@SFNUs!Ebqoa^2B{P@%6x0<3%iF}Yj4pU?LH;dbaKDF zKHy-KSvy)1?dj8>&}h*$6-Qj(L@qQ%CF}F6FrrMD)xz-#8k{KX_a(s!8X*e&^&D^ zykr6&yb$WKApM~LwBxY!5&729KVk_|JPl1XWfBv)In8m%Xs2Y3doehf4j_ zPKA*HJs_NEVtYV>*@jd2(;jfqJ4-*TzkD!&ch_|3;$Df3Sx(62C^T;c6Hy>4yV3xP9?hRltXpf z5Clf-6R&_Y-87oYEm1p6LeA!v;su8*&wXwcK>G%l z@IPqZuqg{w$oo0rO#JqbJmKlKZ|!W#ha79L+1|I!6;Axg{D#sj->mJyy#GL(lVKAl|c?Q9@fHiMzxc39%R#M|LAvZIKE#&|4zv;~_20&wk|Z-?I-Tj5NI! zW@wA2;vr{W?DpOf5G7ZJz*3q;D}kf*E6c@imU7B06Gn&Kr5pT4ln^Gvqr5+#-(S5+e!9u9sQJk zu9ZFXr&@x~RBVJ$MJ&qpc6&J-%BNPeBtxn3#@KOtB7KZ%`RuL2eWqlSAqQUnBb0$+ zJm@);EssftQaSZJOTf?LA_j?jtEepaax&M2)cqrr+uQtJ<4|TFI7l*-A)Wn$A8&o9 zY87(McOEWaW}rCr{-2;k8oG?Um_u3GOEQ%Ac$$SMv`&-NB-F8uJS}D`BPWNh&=c#I z8_ANL5ZIO|M{(|ZsidgmH?Jt!c}@<-hafU!S;qp9G5w1@+9N( zq-lctwjZaSXvOT;Ly=8oz9!rE?Vq4Tu)kM7fkXLB@D0gOTK>Ed((@`cPR9NlXKl;r z!N+BvR{tZEtF$SNaV7pN%$j5<%fjO6_GIXzZ9{{7*f1q@S6&9w{v(up5z)X7J-{z> z8E8p{QX?k+w6(RtzK~VibcOZ>rm;1wGXDsrAXRn59Lmd&Nrp0Po2_YVX;*540)>&z z+v)f3OICjUM<{8{YcdS z_E{?fQ?r`^c2E8hN|c(=IvmOu`fEsrGLTxoJ$Td5`{TZoUzt|VWI6=L?E6P3>62eI z&!H^OCmG6%hx;Ck9(ev?bElT$mmizO>&_?^{Uek(E`f=~OYYa$y$&yJD{+VX`nfIo zdDx$Y$l-j`Q?>*)yxJuT$NTT`Y&(c`h+P>gZWwcbu1Ry3{9ecaKwt$HvtI zy*{CP#Abh@@#f~Q2m%VULD;)cc+e4b2ah1SD1T)x=5xB?ce<62M=~Jxgpz%RwkN-5 z1&fX}JI?&0E>xmrioolB)Ibjju(83iHoHM#!O0if%u3Z72T_T2ovrmc-Dy<*oq;TCeM?j}9y?kR5{6zt0tnA5uh*Zy z;h6o4sbPZk`bn-3IR$*J-P~ogJE%SZU|;)w!(x8jFoZpniDPgFS0wIGxdMAA=V!*{ zr@T)3-^~_Rjp93`qW0c@DZI{_*s)6H_cF0#_Ow27#*FK%$bETc&Rp5OEDS6Z0t)n5 z*s%&8bVQ9+2%D*7P^2qzIy@hH>S1TE-J}C55rS%U6$@|y<({TUJ4GS(i!^iTZ&7mGQw`yv}>N&Z_ja{ z5g3jUg)+Y;=)?8fHt4~e;JyysD87euyM16`fY}7g z3-3ZB&7;f&ZJU)>Cip%7n*P~>YeenE#IIX>d-3f<1->jY|Dv{tpq`svEZtmp-xprt z?R0=^%r8t(*$jQm3b0AT?E_15FqvSxGIxrQ3GfncT7#uJ7r(*M^27*jVae;W; z@=|k!*WB__|G_OmhRAHcXQD1`$I+FAhTo=riFJ@sW4BdgB|P32o0OH_bXWUWQG1?5 zgRyJyPFFFhD*fOJWFU(4RSx1#{22%Hj-_mBWRm`4|83AV$qG_aULf6 zR4wIQYd8-;(1UZrNJjChjZ6=_yabo(kw6+zSVH8wc4toR92QAHP;sq@jXPd*gfzn2rw2Y&p(>8=H~h#sFQGK}w!NM)`nIp8v@kcaA=xk#a?Y4Qs_U$5P2C zZ~ZJH|KwURgHtY-48El$6z{6G^4xS~;PYwJ#z?QWujet;2Z8EZ(#jd0AN0yy!MgU9 z_!Gro?cRX~VtW{GQ^W24(rJu*XZ!ueA7AmYRAlag;5^@YRQt{NHiX!j#-keqtnUUP zPp*R~AQpezTwgwR4D84CFnGWbHNYhZ4t5y6hzBI?@o+YMNa1{J^J?t#!+5|Ewd_TZ znC>y&t9CHo0w7Y&Nq9vMLg7;B8lz-#u~#fq zCrF+aP8Nw>2*M-$Tdgefj~~6`iUSNWHC1O{PTV}Jx2<#%jRzBAYxWwQDScckOA42T zV;*9gU%sE(-N5OuBgbES-Mx`{;p#*p_FVke=GO~7T9;7S^X<^s~I zTci_1V+;5h9-Kt=<^*vPEbGOh#7QVU+ArAN91pnvmc;4Dqukg>a`Av8iaCPBY^<+O z9HXfX(zMIqOG@7x{7Y)U1>%THncG*J&xaozQOpqp*E&?t+L-rxDB6O0mDhUZL6P)h zv&*>OFBi`Yhg$$RQp^zu5>s5YO`Cj8?YqqZdQlxQ;Zgf9_(3-_gP1E2Ub8-HX4gj+ zpvQwt_>X%$oDIEKX5!JVAajc%cf2^pi8AO5r-c;IzKK=mb^oU@il7a;dUwX$%TLxT|$Z0W$tCyFed)cdVC1b78p!hh@) z2xME{4#pBbQA=sCIgq@~=OD|b6n`#yz$@TPliVxdc){xxAfWK#e5u06A`tV$;)|aT z$KSmX zeG_C`%0;eepuvMPO>Bcl&|%jcm-VM&|sVOQX0RK z62B@+-wc#~CcrrLSwj%|NyQ7^;E-E=QG zYeK=)rBI1w&(EfvW>YJnuijc@0Jtymky2QstN?-g-cql$_T2ZDlxpu=e5-r0_t?pe z89!;K?-f3Md7jw%`@MVGHGo5TWGHQ0xRIRW)*6#%@sN^Y65mlR|3hPqOLD=5)V-{E zbzdvlAqpUz$gZ4UH#?Gm8w*ZBa0vt7Te9;W+jWw&p_`U@fZlaGw6vQDC!_d|Cw#zR#>0ZQIdUoi-RRdP{@n2fJvBfB(9NN;!H{0eF#br0Sh8T0WO6c zj@xYgW5cQC_EZryR`D1Xi65iCFv9EA<5_d6MvB!<2Ku&N z0fcm{ptEuK7mDKt1rOcqde7t!eP74|#J_|?@+buMzTJZ`ez-@_IFp%%yyuK23z`-cZATV+=HR0-0GpOsxaVMEd?C)__|MG-JROW=# z4h7c^^ipJhol`hIVr{2658SsS_eFXQW}d>3yJ3*6LF8s4zpU@K=tpl(hxTrJ8oBx2CPG32f217FXqV_-LqN0EKYbPw2Ad_OH7)_ZPxyQ=-MoP3NXWhip6lj=`Q`;YMi z|G%?G$b0zy|IQvE=V!uwWoOu+YVIpL*%ZQ;S?U6Fru{xl4`8;;8ojI}`kln0CJ&oT z?YO%38PXPHTRpxuiGc;TP}i3TEA-aiD*W-SADH35CH%)T z9K=?^pbRGUVmS}aB^KvTQ8y`jI@XtJhypVlIMc+=a0q4_P66)>XThcB;gb5eK>UC3 zm7N9Kz#L+@czC%v1W~IJDE=eLd$P{ZYdor>>y+IQ#`LJh4vmjEwG+25?;HgNIb=>q zuhF-87&3bjEl?c2AhTX#AZ>FPo8w*S+Y+d8E+yy(T)OV4IMMWdlj{7@zlO7o&4t33 z4R70mptPss?~seT2Y_2Ix5ef*tKzhc>n26L)5jBHG{TAgyci`*M?u#OPhPnI2H`AD zmHsA*6zCh{2%-lO%5{0`w0IscG{+B)sD_UqI9TT38@?HzSW`01%O7l1Fv!(GaJd3% zcnO~)X3vTJ_1yx-VjQ~#F`{q9%Q$<@W*QLtn2ldlSU|i7);Nd_1e^liUHJ>#!NZ@O z#ath0<*7Jw0&HPkruxx%(&2h%V0yB~y7@5ZakDAk#rk7CT(lkd(#M6*3|=A81RgH) zXeg$|vw5cmCT(N9rE~Aen79``_nwS>FF$OJ_IeaIA*t%cGsfzMZnq93_SsXv7Wvi8 zk!96%8tJ3nTeL1ou1B=vMKh0!fCBGP?9)zo&=EDMB#5qyBHq|T`;A(N4jrFrD!vuh*pCQZw7J~0 z$rfOPL)PuF!edjao~xm1$x4S`hUg9>hWYN0U#*ZhEN^84-S4d)dfSyz*7uMIW2*Vx zEur?$?yy0>Y$%+dJD8ti$n49TR>zwrIcUyxr~$oz3SK~BP&<*{NTj&)T+cMMs*R>(9_h@du8`$9_>S+?!~0>uZ+x z9kQeTAtxdr}ckX zUeNb@O0-q?@UM|RV0P`Q4WIRiQl1nA=*zM=27r)MqHmuBh z56Tjp$O;z-ge_(tTTE2Pb#b>-rnfg{2j-SCxYI=3&pzC|O^D3v4Y8dCI*2isf=;Zw z=r4w>&LpS+0nA+NDBtD*++KYd zB>H9K;kb^NX0VwDtrT)Z=W1Mj9)ON%`;U81k0q@tX|Nj&&pyD|rWiKlb>nbI#i(19 zs;t)?2s%jG*b_RW6htBZmv+`qQ~T;mAGyiX`aZ0BRb7xs_?`|QHYxl+B;v|L&o!De ztq(rzNM<+iNRhQhCVqd*cXX|TSS6cl;{-qcYjH8k1me-)csPQ=@%=DAa6F7`0uu%f z;sc*RlT&c#pX^10E&B(^?#gq|fA2W* zW3pvWE`P?CG3eXr!OB>ff@xM62b0Dz7zY!lKqpPLuQZ%~QtWQ7jjJ>-;BW8A56+xB zx90NRB71pLD*+wRLe9*48#Op~xY#mXXFGNo9r&)-&i{hir;|6Hi6R}?@2%cY+3@2G zxfKoVTfc#46hVge$M)s4^Ef?Y|=jFn=qO<66D-Nb!J~|Y%(>Kkr4G4&yVHm)e^w% ze4PW2F^9(u)Q`&rNFT^X2}NAWebs+S1wFV*zT0l#tt&d_^VHg>NgwG;X3h4bnw3dy z*(jcEo+|m1{zKD|eLYob9vDZ)} zgQzjvn-`(nbs2}M!UNUqP1!yq`K_&@h2*xk;CppMR^*Q%1LN`b=%LIDY`za63Lt9u z`*+Lmq6N!T@h=+@wPs7yve7CFW`nyLku?A@a3cd^i6Lm4AafvM5cA-$%Z+ds0_aG) z+z3}}IryFeXk~s{)o$D@k7wUAVgWulOY<^)xx$WOcdwNu{;?+WVbCM_zVP>fO8mh> z)XOe6&T{9y_V7kBFy|Lwdqaj3uM2RspOIq-SNjd#(ie#6sa%!GYf<(JY;&BD@682x zBWkAi(#%51=K9E{^*?9dp^Q0Qxr~5~%a>v*Y2F0nya#fv!D8N-f=unTO zL-W6~k0GxC{DyUv?%dl-Nb&Kibi6UtG&F-~aRX4n!Sjl)im8`?iaV6)S#t&|5cO}fyFJL)Zo#RAatJih#*hrmoK(mTS-1*#l!l1XPD@Y&Sp7cJ1)4) zLFggq)1WNbZ=R8VD|shm1n>&Dg#Xwp=p@^EYc~cw=?H2BnFEWIXDN4{vGyiQ1iS*y zG|9aJju!%60mi#j{;O1%2LE-wKrEj4g!GPL`e4+S6lPEr=j z7rB=uO!Q0n50zgxP&H?%j!VTP$5I6%te$jM955tN{AAb ziYOG4Kc8Y}9( z{|1Gu9!@hjFh5lE;L0ScCm0sojTAR5IME>)76wCtH;dxOpPV?Z3vSXBO+)8w8xgtX z?0nm@sj?ZwBGsNDhGTB|vpj!dVvNBMPs0n0UOXN2cp45skKzZJwbn7|M9#%{7C|? zX_I9n(}Sxa+1ZdL^G_e`76}n8xa&X}&pM{@rjGwE zlG_adr|hvWb@ONKO0J}P_U%#CIr^*<<&X++B7*wcnLav6$yRqy+&pIzY`ma@fxVs& zDe^}W3(9q%`Gg1v-$*#c78gt7D(*zz)64(MSZPgQv}T!Y()&V4I|ZH@tJaVB$7gPZ``X8 z^~~akIrrFKfRT~4Haf5t({S5-saKF6otu@~g5iR)(>W0MU7qrf*~WLC_O;`#YrQ2O zbdN?%1yTV7n?w@f3w+T8;^lL!^1QyERZ2_du>1<8^)Ut7SRS5Y{2i`37d31QHEsO#m|z^U1)|ZUwwXT+!aM5BX25zx($kZSlmsN9*Z6Y62r2 zS)UTD4V-^C$+Cq**-PF-`&xC*fajVvh#pX1tY(0v{V2XKfkOUvM04DI+yu|-BUs+C zHk1A{k%gTtzYfkaQL6jKRXPGV1j&(Q;;52CMGq%AfSSI!5?CGnxdPDizzcx{6w)(XlIukBsL*Y#D7!hDHEN~7m2*Pg{6q^D2j-2cTu@~xjU4(% zhjywhp^(+XX$Arg6+H+XhGg{w!-Bh!;)Vq$I>a}Xy4wb$k|CKTU0}#yngkzQMxr@l z$5`!f+XJ#AyZT9G(4(SW@!y$`bG8eAM>5>Wa>bg-8Cs&heR77DzVXT@Iq-y>+k|uR zmH^pj*rDz<0xW>v1R^8 zAV7a86mo!6=sKYlf1h9hJ-|T&a^(gyJnH2} zp6Y0ZuRK04ydmP~(d3O2>W0F(4jZK6dGGE1ZEzT*a2!_9?R#m7G!S2*J_zVR=7A*T zKK@OdPrKq`&&0P_+Sk%IrHwQ~(@#Ki-ce^kun?EggzmX&xnu(%MNw1UxZLhR@vPm8 zMsDps?^wa|QI=fyVBnsl%UkArJ%KkF${3{xWxBzj6o{Bn(SyMOuY(P$SL_Juya?dP z^4L+q!AcH!{u`mXp4T-&a1Y>GMRE544&7{i=cMeA;J^EJFy1Jovl=RI!Wnfh4}RCc z(DnMbQ2PGRe_x+0;Vels6PE1x8|gZ+oNAd8auz39yLZ7g9*4x6WYmOAzzu~P%0~$w zy=)YtpY4lHve5LWM_c-EFodkB>Q=$;FDkpH?b(QkhFYg$tX;SqLtHA{_-P(-tqmcp zNR~a1%+uiGZ_zB<;nEY(dOE{qSD!~D1O;pk6OuuY%JI}20cpvVBoMwVO*m&ZP_{(Q zsq{)!um|?bPPP33*XehVJ1XpS+WjIf+`EK>%TDmVgv_0bGG!VNpx);q)Rw2M?qt26 z7^m2fxnT3+J@$|a80e2NP2QVQp(#|}uu_ELnLeZYepsZt-cSKZb%J~FFQ+<9ayTx{ zqC`xCnPxamX@6xIqEUOs3Zy#0u?Cbz*#tN^xy+jCMAX_eUdmEag8u{CjZE<-j5G4; z%PBU_DBN@N##D%uZJ4OqEG5+B-dC_gABQBjrFUH&^s4}$gVr5@p=f*&Piqf!U~?a- zU}4i67NTl*6W1}O)UOP_sX}fCw%y&vbv9CDNn944^2VK7YSLF;B|vcKtzvD zFcO<Tn0mqu+RskoMS*>CkC;~;$D#9gVF+s4iT_;>4dUi^SE>X<0`%)tA zKA3~s8ITZCv7$6=onLXPEe?zV)*U8!a7f3=l{yu7mLIXSH)(e@mxUN~FSTpFe<9zr zoxb(~xhr)twxWA|Zx()1I_1H7_12r$S{Ijog#ZPECebJr=+%j_G>t+y<&(K-dBe7Y*}JAYN?5}G1O={ zFenw~mk>H70y?rfB`P{Q_Qdpi*L%{Hv?qKLuT>>P>@}A833Qa{l>Y^~pXrpYyzTCq z+&S!?yb8}=Y)Dkt#qGEvD^Kgx%}uGkeMh`%U`&Ttgnhj|mlpV{h_HOV?P>wUl7p;bdxf?1MQpH6%)!X_ch3B?^ju7G$vq4nX zXw;{`+N-KYlZy=6S`Yuw_ElIG+KJj+-EqLnM;O7#vT4{i46w?w8HV-bGA%K8u9=@c zFgz+#zVR~;qL4t~x<*9Hh1OiocP;Vi>EZs~xkaT{GvC#;ZMY` zZ2YWoDT^g?4HCFJrJG+%p8ygh0cR8L!N1{bQbkpc_)uBBTn!f76pej+@_kVj*6~Bw zWx&~lV@+{q6HYG3&L&A~Q-~=`Obecnaj4x?O0gauN~a=N_OmKZa$?tH zrY#$tu~vS^WPl+c^`4+dyH$)@Uqw5+Gq1*@?7sA+a~}^)%MKp? z#6*hNxp7>Wf9r zjGEvM=!v~8pPXm)pFQ|P^DH&v`A14zee36Ho=r*W?2U=K`D9+*6`O`{3CnY}0$g!m z^?_*+xNis5?UcohF|Fo>Pvts2C4~1^nvpvKpz?MwEA3?7r)9iuG{v>aZ}dZr-pL3g za0c6QLNn|8i1RGmgMVY5U9BnFH^R1N2;mzK7cQC;*)@%Wyo7Bc;aF3Ao`sVO@;pn@ z8d#GQq^ATwxepg{)ZiM2Bk>>{QEhsNbue&yNt=WeCy~CQx|QIZ4A_`5PaV!YU#JvBw8XQ&H&osk`sjHTBi;U+cQZ+s4ku+eX}JpSv63nZPs$t35K; z_EE)Cf0@st!?;AUaxViH)rv=|yA>Frp9#V=5KG+9(e#woZ*Azv!|;U0z}kXwi>#Eq z-9HjHcicFCE-c_}CQHi8Z=8A=&w~i>dZw=o$_Ym^Nz-yXjUhl;bk+W$f8)iRe7T?2 zgpX2wDe51)(iD~niuh=@th!&(aqF9swnO8KcQ_bzOcqUpJC)xU`MO15{ejU&zVfz1 zh2xhh$WDww<&FBstRSF@v|nY_kAxkO0^B3G2mi8rM5Dc+sO_fIT4mKF5IZnAzwitF z`G>>oM*;T;jy1XN5vtPQkzL0om#7GSZCE!dS^6k<}VMi;S8 zQ%J*y+!qU%sPC};N5=5LC?1f{AGs9@ohVG|Mv|IbsBbY;zc2wJu8?#4lP|iRDY!N zcsis4T&xfJEjvZG6#dbBJXRf0!``<$@z_m(St?G%>S;U_?V^5;;~n8Ert$B(xQX1n z8<8McubVFKYo7LR zWR$$m@8etu4VJ*5FAl;p=>c+dTNH+8QUgZnLV<7{>Q(buO7FeS&MxB&--WxPK<+36 z!v~rP(P~8V*6eWb=kL$dC6h0thWH+nb(&3%RHrj?2%Zlmz@hS14CP-r7HhbxMt<}* z=a6IRXLZI|!C~ZV`3}t$IN3qvfZRicr@9!A@;0|?eMneFeQq|%VFc%C2n2crq&4OIJiPm8Mb_H?_7af{yzKV%MMvST$0V*kq7n?Y7M=8vx1vT$&qby z7}K2Uxsn|DHpepHJ6GUHrUX9DuGTfr?B)ubT1HdhT4wuvH?Hd{MzNBctL5Ic&kk-5 zd+Yz(Tr%gXAv3)p!Rx&crJ&9bU|S>6kSCA|r;fAQ99da7sO}o7!#4%W@K$46$!9LHR@lCajEeu|j5EKrYL#&R@-)h`H zWbZBzSXC4#IUx$EFb$JPQ8_W`GvTS50i2#p*YVVKfZAp5V835-wG6IFq_r$MRS}=^ zFfNnaw=*57{rp@T*;(aRGWJ^wyVyc4w(RHX+w4_3aE`Epud zW2Mq|(OeuD5Q(NbdTg|#HgDVS@yXEVsG+!CVkl4*Q(#f>* k$z>i|%?5N^IMx*Jws3O!^>7kNt20_fJOv9D?mD>ae?=8nIRF3v literal 0 HcmV?d00001 diff --git a/core/staking_verifier_test.go b/core/staking_verifier_test.go index f829a00500..eeb372889b 100644 --- a/core/staking_verifier_test.go +++ b/core/staking_verifier_test.go @@ -1651,7 +1651,7 @@ func makeVWrapperByIndex(index int) staking.ValidatorWrapper { } func newTestStateDB() (*state.DB, error) { - return state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + return state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) } // makeVWrappersForStake makes the default staking.ValidatorWrappers for diff --git a/core/state/access_list.go b/core/state/access_list.go new file mode 100644 index 0000000000..4194691345 --- /dev/null +++ b/core/state/access_list.go @@ -0,0 +1,136 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "github.com/ethereum/go-ethereum/common" +) + +type accessList struct { + addresses map[common.Address]int + slots []map[common.Hash]struct{} +} + +// ContainsAddress returns true if the address is in the access list. +func (al *accessList) ContainsAddress(address common.Address) bool { + _, ok := al.addresses[address] + return ok +} + +// Contains checks if a slot within an account is present in the access list, returning +// separate flags for the presence of the account and the slot respectively. +func (al *accessList) Contains(address common.Address, slot common.Hash) (addressPresent bool, slotPresent bool) { + idx, ok := al.addresses[address] + if !ok { + // no such address (and hence zero slots) + return false, false + } + if idx == -1 { + // address yes, but no slots + return true, false + } + _, slotPresent = al.slots[idx][slot] + return true, slotPresent +} + +// newAccessList creates a new accessList. +func newAccessList() *accessList { + return &accessList{ + addresses: make(map[common.Address]int), + } +} + +// Copy creates an independent copy of an accessList. +func (a *accessList) Copy() *accessList { + cp := newAccessList() + for k, v := range a.addresses { + cp.addresses[k] = v + } + cp.slots = make([]map[common.Hash]struct{}, len(a.slots)) + for i, slotMap := range a.slots { + newSlotmap := make(map[common.Hash]struct{}, len(slotMap)) + for k := range slotMap { + newSlotmap[k] = struct{}{} + } + cp.slots[i] = newSlotmap + } + return cp +} + +// AddAddress adds an address to the access list, and returns 'true' if the operation +// caused a change (addr was not previously in the list). +func (al *accessList) AddAddress(address common.Address) bool { + if _, present := al.addresses[address]; present { + return false + } + al.addresses[address] = -1 + return true +} + +// AddSlot adds the specified (addr, slot) combo to the access list. +// Return values are: +// - address added +// - slot added +// For any 'true' value returned, a corresponding journal entry must be made. +func (al *accessList) AddSlot(address common.Address, slot common.Hash) (addrChange bool, slotChange bool) { + idx, addrPresent := al.addresses[address] + if !addrPresent || idx == -1 { + // Address not present, or addr present but no slots there + al.addresses[address] = len(al.slots) + slotmap := map[common.Hash]struct{}{slot: {}} + al.slots = append(al.slots, slotmap) + return !addrPresent, true + } + // There is already an (address,slot) mapping + slotmap := al.slots[idx] + if _, ok := slotmap[slot]; !ok { + slotmap[slot] = struct{}{} + // Journal add slot change + return false, true + } + // No changes required + return false, false +} + +// DeleteSlot removes an (address, slot)-tuple from the access list. +// This operation needs to be performed in the same order as the addition happened. +// This method is meant to be used by the journal, which maintains ordering of +// operations. +func (al *accessList) DeleteSlot(address common.Address, slot common.Hash) { + idx, addrOk := al.addresses[address] + // There are two ways this can fail + if !addrOk { + panic("reverting slot change, address not present in list") + } + slotmap := al.slots[idx] + delete(slotmap, slot) + // If that was the last (first) slot, remove it + // Since additions and rollbacks are always performed in order, + // we can delete the item without worrying about screwing up later indices + if len(slotmap) == 0 { + al.slots = al.slots[:idx] + al.addresses[address] = -1 + } +} + +// DeleteAddress removes an address from the access list. This operation +// needs to be performed in the same order as the addition happened. +// This method is meant to be used by the journal, which maintains ordering of +// operations. +func (al *accessList) DeleteAddress(address common.Address) { + delete(al.addresses, address) +} diff --git a/core/state/database.go b/core/state/database.go index 9144b45da8..d6a0e7e15b 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -17,24 +17,23 @@ package state import ( + "errors" "fmt" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" - lru "github.com/hashicorp/golang-lru" + "github.com/harmony-one/harmony/core/rawdb" ) -// MaxTrieCacheGen limit after which to evict trie nodes from memory. -var MaxTrieCacheGen = uint16(120) - const ( - // Number of past tries to keep. This value is chosen such that - // reasonable chain reorg depths will hit an existing trie. - maxPastTries = 12 - // Number of codehash->size associations to keep. codeSizeCacheSize = 100000 + + // Cache size granted for caching clean code. + codeCacheSize = 64 * 1024 * 1024 ) // Database wraps access to tries and contract code. @@ -43,7 +42,7 @@ type Database interface { OpenTrie(root common.Hash) (Trie, error) // OpenStorageTrie opens the storage trie of an account. - OpenStorageTrie(addrHash, root common.Hash) (Trie, error) + OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (Trie, error) // CopyTrie returns an independent copy of the given trie. CopyTrie(Trie) Trie @@ -54,6 +53,9 @@ type Database interface { // ContractCodeSize retrieves a particular contracts code's size. ContractCodeSize(addrHash, codeHash common.Hash) (int, error) + // DiskDB returns the underlying key-value disk database. + DiskDB() ethdb.KeyValueStore + // TrieDB retrieves the low level trie database used for data storage. TrieDB() *trie.Database } @@ -63,7 +65,7 @@ type Trie interface { // GetKey returns the sha3 preimage of a hashed key that was previously used // to store a value. // - // TODO(fjl): remove this when SecureTrie is removed + // TODO(fjl): remove this when StateTrie is removed GetKey([]byte) []byte // TryGet returns the value for key stored in the trie. The value bytes must @@ -71,23 +73,43 @@ type Trie interface { // trie.MissingNodeError is returned. TryGet(key []byte) ([]byte, error) + // TryGetAccount abstracts an account read from the trie. It retrieves the + // account blob from the trie with provided account address and decodes it + // with associated decoding algorithm. If the specified account is not in + // the trie, nil will be returned. If the trie is corrupted(e.g. some nodes + // are missing or the account blob is incorrect for decoding), an error will + // be returned. + TryGetAccount(address common.Address) (*types.StateAccount, error) + // TryUpdate associates key with value in the trie. If value has length zero, any // existing value is deleted from the trie. The value bytes must not be modified // by the caller while they are stored in the trie. If a node was not found in the // database, a trie.MissingNodeError is returned. TryUpdate(key, value []byte) error + // TryUpdateAccount abstracts an account write to the trie. It encodes the + // provided account object with associated algorithm and then updates it + // in the trie with provided address. + TryUpdateAccount(address common.Address, account *types.StateAccount) error + // TryDelete removes any existing value for key from the trie. If a node was not // found in the database, a trie.MissingNodeError is returned. TryDelete(key []byte) error + // TryDeleteAccount abstracts an account deletion from the trie. + TryDeleteAccount(address common.Address) error + // Hash returns the root hash of the trie. It does not write to the database and // can be used even if the trie doesn't have one. Hash() common.Hash - // Commit writes all nodes to the trie's memory database, tracking the internal - // and external (for account tries) references. - Commit(onleaf trie.LeafCallback) (common.Hash, error) + // Commit collects all dirty nodes in the trie and replace them with the + // corresponding node hash. All collected nodes(including dirty leaves if + // collectLeaf is true) will be encapsulated into a nodeset for return. + // The returned nodeset can be nil if the trie is clean(nothing to commit). + // Once the trie is committed, it's not usable anymore. A new trie must + // be created with new root and updated trie database for following usage + Commit(collectLeaf bool) (common.Hash, *trie.NodeSet) // NodeIterator returns an iterator that returns nodes of the trie. Iteration // starts at the key after the given start key. @@ -105,41 +127,66 @@ type Trie interface { // NewDatabase creates a backing store for state. The returned database is safe for // concurrent use, but does not retain any recent trie nodes in memory. To keep some -// historical state in memory, use the NewDatabaseWithCache constructor. +// historical state in memory, use the NewDatabaseWithConfig constructor. func NewDatabase(db ethdb.Database) Database { - return NewDatabaseWithCache(db, 0) + return NewDatabaseWithConfig(db, nil) } -// NewDatabaseWithCache creates a backing store for state. The returned database +func NewDatabaseWithCache(db ethdb.Database, cache int) Database { + return NewDatabaseWithConfig(db, nil) +} + +// NewDatabaseWithConfig creates a backing store for state. The returned database // is safe for concurrent use and retains a lot of collapsed RLP trie nodes in a // large memory cache. -func NewDatabaseWithCache(db ethdb.Database, cache int) Database { - csc, _ := lru.New(codeSizeCacheSize) +func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { return &cachingDB{ - db: trie.NewDatabaseWithCache(db, cache), - codeSizeCache: csc, + disk: db, + codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), + codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), + triedb: trie.NewDatabaseWithConfig(db, config), + } +} + +// NewDatabaseWithNodeDB creates a state database with an already initialized node database. +func NewDatabaseWithNodeDB(db ethdb.Database, triedb *trie.Database) Database { + return &cachingDB{ + disk: db, + codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), + codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), + triedb: triedb, } } type cachingDB struct { - db *trie.Database - codeSizeCache *lru.Cache + disk ethdb.KeyValueStore + codeSizeCache *lru.Cache[common.Hash, int] + codeCache *lru.SizeConstrainedCache[common.Hash, []byte] + triedb *trie.Database } // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { - return trie.NewSecure(root, db.db) + tr, err := trie.NewStateTrie(trie.StateTrieID(root), db.triedb) + if err != nil { + return nil, err + } + return tr, nil } // OpenStorageTrie opens the storage trie of an account. -func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { - return trie.NewSecure(root, db.db) +func (db *cachingDB) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (Trie, error) { + tr, err := trie.NewStateTrie(trie.StorageTrieID(stateRoot, addrHash, root), db.triedb) + if err != nil { + return nil, err + } + return tr, nil } // CopyTrie returns an independent copy of the given trie. func (db *cachingDB) CopyTrie(t Trie) Trie { switch t := t.(type) { - case *trie.SecureTrie: + case *trie.StateTrie: return t.Copy() default: panic(fmt.Errorf("unknown trie type %T", t)) @@ -148,23 +195,51 @@ func (db *cachingDB) CopyTrie(t Trie) Trie { // ContractCode retrieves a particular contract's code. func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) { - code, err := db.db.Node(codeHash) - if err == nil { + code, _ := db.codeCache.Get(codeHash) + if len(code) > 0 { + return code, nil + } + code = rawdb.ReadCode(db.disk, codeHash) + if len(code) > 0 { + db.codeCache.Add(codeHash, code) + db.codeSizeCache.Add(codeHash, len(code)) + return code, nil + } + return nil, errors.New("not found") +} + +// ContractCodeWithPrefix retrieves a particular contract's code. If the +// code can't be found in the cache, then check the existence with **new** +// db scheme. +func (db *cachingDB) ContractCodeWithPrefix(addrHash, codeHash common.Hash) ([]byte, error) { + code, _ := db.codeCache.Get(codeHash) + if len(code) > 0 { + return code, nil + } + code = rawdb.ReadCodeWithPrefix(db.disk, codeHash) + if len(code) > 0 { + db.codeCache.Add(codeHash, code) db.codeSizeCache.Add(codeHash, len(code)) + return code, nil } - return code, err + return nil, errors.New("not found") } // ContractCodeSize retrieves a particular contracts code's size. func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, error) { if cached, ok := db.codeSizeCache.Get(codeHash); ok { - return cached.(int), nil + return cached, nil } code, err := db.ContractCode(addrHash, codeHash) return len(code), err } +// DiskDB returns the underlying key-value disk database. +func (db *cachingDB) DiskDB() ethdb.KeyValueStore { + return db.disk +} + // TrieDB retrieves any intermediate trie-node caching layer. func (db *cachingDB) TrieDB() *trie.Database { - return db.db + return db.triedb } diff --git a/core/state/dump.go b/core/state/dump.go index f429ae1077..d9e031ed4a 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -17,29 +17,27 @@ package state import ( - "bytes" "encoding/json" "fmt" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/harmony-one/harmony/internal/utils" ) -// DumpConfig is a set of options to control what portions of the statewill be +// DumpConfig is a set of options to control what portions of the state will be // iterated and collected. type DumpConfig struct { SkipCode bool SkipStorage bool OnlyWithAddresses bool - HoldStorage bool Start []byte End []byte - StateStart []byte - StateEnd []byte Max uint64 } @@ -48,10 +46,7 @@ type DumpCollector interface { // OnRoot is called with the state root OnRoot(common.Hash) // OnAccount is called once for each account in the trie - OnAccountStart(common.Address, DumpAccount) - // OnAccount is called once for each account in the trie - OnAccountState(_ common.Address, StateSecureKey hexutil.Bytes, key, value []byte) - OnAccountEnd(common.Address, DumpAccount) + OnAccount(common.Address, DumpAccount) } // DumpAccount represents an account in the state. @@ -78,15 +73,7 @@ func (d *Dump) OnRoot(root common.Hash) { } // OnAccount implements DumpCollector interface -func (d *Dump) OnAccountStart(addr common.Address, account DumpAccount) { -} - -// OnAccount implements DumpCollector interface -func (d *Dump) OnAccountState(_ common.Address, StateSecureKey hexutil.Bytes, key, value []byte) { -} - -// OnAccount implements DumpCollector interface -func (d *Dump) OnAccountEnd(addr common.Address, account DumpAccount) { +func (d *Dump) OnAccount(addr common.Address, account DumpAccount) { d.Accounts[addr] = account } @@ -103,15 +90,7 @@ func (d *IteratorDump) OnRoot(root common.Hash) { } // OnAccount implements DumpCollector interface -func (d *IteratorDump) OnAccountStart(addr common.Address, account DumpAccount) { -} - -// OnAccount implements DumpCollector interface -func (d *IteratorDump) OnAccountState(_ common.Address, StateSecureKey hexutil.Bytes, key, value []byte) { -} - -// OnAccount implements DumpCollector interface -func (d *IteratorDump) OnAccountEnd(addr common.Address, account DumpAccount) { +func (d *IteratorDump) OnAccount(addr common.Address, account DumpAccount) { d.Accounts[addr] = account } @@ -121,15 +100,7 @@ type iterativeDump struct { } // OnAccount implements DumpCollector interface -func (d iterativeDump) OnAccountStart(addr common.Address, account DumpAccount) { -} - -// OnAccount implements DumpCollector interface -func (d iterativeDump) OnAccountState(_ common.Address, StateSecureKey hexutil.Bytes, key, value []byte) { -} - -// OnAccount implements DumpCollector interface -func (d iterativeDump) OnAccountEnd(addr common.Address, account DumpAccount) { +func (d iterativeDump) OnAccount(addr common.Address, account DumpAccount) { dumpAccount := &DumpAccount{ Balance: account.Balance, Nonce: account.Nonce, @@ -169,15 +140,9 @@ func (s *DB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []byte) log.Info("Trie dumping started", "root", s.trie.Hash()) c.OnRoot(s.trie.Hash()) - hasEnd := len(conf.End) > 0 - stateStart := conf.StateStart - hasStateEnd := len(conf.StateEnd) > 0 it := trie.NewIterator(s.trie.NodeIterator(conf.Start)) for it.Next() { - if hasEnd && bytes.Compare(it.Key, conf.End) >= 0 { - break - } - var data Account + var data types.StateAccount if err := rlp.DecodeBytes(it.Value, &data); err != nil { panic(err) } @@ -202,29 +167,24 @@ func (s *DB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []byte) if !conf.SkipCode { account.Code = obj.Code(s.db) } - c.OnAccountStart(addr, account) if !conf.SkipStorage { account.Storage = make(map[common.Hash]string) - storageIt := trie.NewIterator(obj.getTrie(s.db).NodeIterator(stateStart)) + tr, err := obj.getTrie(s.db) + if err != nil { + utils.Logger().Error().Err(err).Msg("Failed to load storage trie") + continue + } + storageIt := trie.NewIterator(tr.NodeIterator(nil)) for storageIt.Next() { - if hasStateEnd && bytes.Compare(storageIt.Key, conf.StateEnd) >= 0 { - break - } - key := s.trie.GetKey(storageIt.Key) - c.OnAccountState(addr, storageIt.Key, key, storageIt.Value) - if conf.HoldStorage { - _, content, _, err := rlp.Split(storageIt.Value) - if err != nil { - log.Error("Failed to decode the value returned by iterator", "error", err) - continue - } - account.Storage[common.BytesToHash(s.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(content) + _, content, _, err := rlp.Split(storageIt.Value) + if err != nil { + utils.Logger().Error().Err(err).Msg("Failed to decode the value returned by iterator") + continue } + account.Storage[common.BytesToHash(s.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(content) } - stateStart = nil - hasStateEnd = false } - c.OnAccountEnd(addr, account) + c.OnAccount(addr, account) accounts++ if time.Since(logged) > 8*time.Second { log.Info("Trie dumping in progress", "at", it.Key, "accounts", accounts, @@ -239,7 +199,7 @@ func (s *DB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []byte) } } if missingPreimages > 0 { - log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages) + utils.Logger().Warn().Int("missing", missingPreimages).Msg("Dump incomplete due to missing preimages") } log.Info("Trie dumping complete", "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) diff --git a/core/state/iterator.go b/core/state/iterator.go new file mode 100644 index 0000000000..dea2c81b03 --- /dev/null +++ b/core/state/iterator.go @@ -0,0 +1,155 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "bytes" + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +// NodeIterator is an iterator to traverse the entire state trie post-order, +// including all of the contract code and contract state tries. +type NodeIterator struct { + state *DB // State being iterated + + stateIt trie.NodeIterator // Primary iterator for the global state trie + dataIt trie.NodeIterator // Secondary iterator for the data trie of a contract + + accountHash common.Hash // Hash of the node containing the account + codeHash common.Hash // Hash of the contract source code + code []byte // Source code associated with a contract + + Hash common.Hash // Hash of the current entry being iterated (nil if not standalone) + Parent common.Hash // Hash of the first full ancestor node (nil if current is the root) + + Error error // Failure set in case of an internal error in the iterator +} + +// NewNodeIterator creates an post-order state node iterator. +func NewNodeIterator(state *DB) *NodeIterator { + return &NodeIterator{ + state: state, + } +} + +// Next moves the iterator to the next node, returning whether there are any +// further nodes. In case of an internal error this method returns false and +// sets the Error field to the encountered failure. +func (it *NodeIterator) Next() bool { + // If the iterator failed previously, don't do anything + if it.Error != nil { + return false + } + // Otherwise step forward with the iterator and report any errors + if err := it.step(); err != nil { + it.Error = err + return false + } + return it.retrieve() +} + +// step moves the iterator to the next entry of the state trie. +func (it *NodeIterator) step() error { + // Abort if we reached the end of the iteration + if it.state == nil { + return nil + } + // Initialize the iterator if we've just started + if it.stateIt == nil { + it.stateIt = it.state.trie.NodeIterator(nil) + } + // If we had data nodes previously, we surely have at least state nodes + if it.dataIt != nil { + if cont := it.dataIt.Next(true); !cont { + if it.dataIt.Error() != nil { + return it.dataIt.Error() + } + it.dataIt = nil + } + return nil + } + // If we had source code previously, discard that + if it.code != nil { + it.code = nil + return nil + } + // Step to the next state trie node, terminating if we're out of nodes + if cont := it.stateIt.Next(true); !cont { + if it.stateIt.Error() != nil { + return it.stateIt.Error() + } + it.state, it.stateIt = nil, nil + return nil + } + // If the state trie node is an internal entry, leave as is + if !it.stateIt.Leaf() { + return nil + } + // Otherwise we've reached an account node, initiate data iteration + var account Account + if err := rlp.Decode(bytes.NewReader(it.stateIt.LeafBlob()), &account); err != nil { + return err + } + dataTrie, err := it.state.db.OpenStorageTrie(it.state.originalRoot, common.BytesToHash(it.stateIt.LeafKey()), account.Root) + if err != nil { + return err + } + it.dataIt = dataTrie.NodeIterator(nil) + if !it.dataIt.Next(true) { + it.dataIt = nil + } + if !bytes.Equal(account.CodeHash, types.EmptyCodeHash.Bytes()) { + it.codeHash = common.BytesToHash(account.CodeHash) + addrHash := common.BytesToHash(it.stateIt.LeafKey()) + it.code, err = it.state.db.ContractCode(addrHash, common.BytesToHash(account.CodeHash)) + if err != nil { + return fmt.Errorf("code %x: %v", account.CodeHash, err) + } + } + it.accountHash = it.stateIt.Parent() + return nil +} + +// retrieve pulls and caches the current state entry the iterator is traversing. +// The method returns whether there are any more data left for inspection. +func (it *NodeIterator) retrieve() bool { + // Clear out any previously set values + it.Hash = common.Hash{} + + // If the iteration's done, return no available data + if it.state == nil { + return false + } + // Otherwise retrieve the current entry + switch { + case it.dataIt != nil: + it.Hash, it.Parent = it.dataIt.Hash(), it.dataIt.Parent() + if it.Parent == (common.Hash{}) { + it.Parent = it.accountHash + } + case it.code != nil: + it.Hash, it.Parent = it.codeHash, it.accountHash + case it.stateIt != nil: + it.Hash, it.Parent = it.stateIt.Hash(), it.stateIt.Parent() + } + return true +} diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go new file mode 100644 index 0000000000..50374059b7 --- /dev/null +++ b/core/state/iterator_test.go @@ -0,0 +1,142 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/harmony-one/harmony/core/rawdb" +) + +// testAccount is the data associated with an account used by the state tests. +type testAccount struct { + address common.Address + balance *big.Int + nonce uint64 + code []byte +} + +// makeTestState create a sample test state to test node-wise reconstruction. +func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) { + // Create an empty state + db := rawdb.NewMemoryDatabase() + sdb := NewDatabase(db) + state, _ := New(common.Hash{}, sdb, nil) + + // Fill it with some arbitrary data + var accounts []*testAccount + for i := byte(0); i < 96; i++ { + obj := state.GetOrNewStateObject(common.BytesToAddress([]byte{i})) + acc := &testAccount{address: common.BytesToAddress([]byte{i})} + + obj.AddBalance(big.NewInt(int64(11 * i))) + acc.balance = big.NewInt(int64(11 * i)) + + obj.SetNonce(uint64(42 * i)) + acc.nonce = uint64(42 * i) + + if i%3 == 0 { + obj.SetCode(crypto.Keccak256Hash([]byte{i, i, i, i, i}), []byte{i, i, i, i, i}) + acc.code = []byte{i, i, i, i, i} + } + if i%5 == 0 { + for j := byte(0); j < 5; j++ { + hash := crypto.Keccak256Hash([]byte{i, i, i, i, i, j, j}) + obj.SetState(sdb, hash, hash) + } + } + state.updateStateObject(obj) + accounts = append(accounts, acc) + } + root, _ := state.Commit(false) + + // Return the generated state + return db, sdb, root, accounts +} + +// Tests that the node iterator indeed walks over the entire database contents. +func TestNodeIteratorCoverage(t *testing.T) { + // Create some arbitrary test state to iterate + db, sdb, root, _ := makeTestState() + sdb.TrieDB().Commit(root, false) + + state, err := New(root, sdb, nil) + if err != nil { + t.Fatalf("failed to create state trie at %x: %v", root, err) + } + // Gather all the node hashes found by the iterator + hashes := make(map[common.Hash]struct{}) + for it := NewNodeIterator(state); it.Next(); { + if it.Hash != (common.Hash{}) { + hashes[it.Hash] = struct{}{} + } + } + // Check in-disk nodes + var ( + seenNodes = make(map[common.Hash]struct{}) + seenCodes = make(map[common.Hash]struct{}) + ) + it := db.NewIterator(nil, nil) + for it.Next() { + ok, hash := isTrieNode(sdb.TrieDB().Scheme(), it.Key(), it.Value()) + if !ok { + continue + } + seenNodes[hash] = struct{}{} + } + it.Release() + + // Check in-disk codes + it = db.NewIterator(nil, nil) + for it.Next() { + ok, hash := rawdb.IsCodeKey(it.Key()) + if !ok { + continue + } + if _, ok := hashes[common.BytesToHash(hash)]; !ok { + t.Errorf("state entry not reported %x", it.Key()) + } + seenCodes[common.BytesToHash(hash)] = struct{}{} + } + it.Release() + + // Cross check the iterated hashes and the database/nodepool content + for hash := range hashes { + _, ok := seenNodes[hash] + if !ok { + _, ok = seenCodes[hash] + } + if !ok { + t.Errorf("failed to retrieve reported node %x", hash) + } + } +} + +// isTrieNode is a helper function which reports if the provided +// database entry belongs to a trie node or not. +func isTrieNode(scheme string, key, val []byte) (bool, common.Hash) { + if scheme == rawdb.HashScheme { + if len(key) == common.HashLength { + return true, common.BytesToHash(key) + } + } + return false, common.Hash{} +} diff --git a/core/state/journal.go b/core/state/journal.go index 35198216da..48dd66f665 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -34,14 +34,14 @@ type journalEntry interface { } // journal contains the list of state modifications applied since the last state -// commit. These are tracked to be able to be reverted in case of an execution -// exception or revertal request. +// commit. These are tracked to be able to be reverted in the case of an execution +// exception or request for reversal. type journal struct { entries []journalEntry // Current changes tracked by the journal dirties map[common.Address]int // Dirty accounts and the number of changes } -// newJournal create a new initialized journal. +// newJournal creates a new initialized journal. func newJournal() *journal { return &journal{ dirties: make(map[common.Address]int), @@ -91,7 +91,8 @@ type ( account *common.Address } resetObjectChange struct { - prev *Object + prev *Object + prevdestruct bool } suicideChange struct { account *common.Address @@ -134,8 +135,30 @@ type ( touchChange struct { account *common.Address } + // Changes to the access list + accessListAddAccountChange struct { + address *common.Address + } + accessListAddSlotChange struct { + address *common.Address + slot *common.Hash + } + + transientStorageChange struct { + account *common.Address + key, prevalue common.Hash + } ) +// dirtied returns the Ethereum address modified by this journal entry. +func (v validatorWrapperChange) dirtied() *common.Address { + return v.address +} + +// revert undoes the changes introduced by this journal entry. +func (v validatorWrapperChange) revert(*DB) { +} + func (ch createObjectChange) revert(s *DB) { delete(s.stateObjects, *ch.account) delete(s.stateObjectsDirty, *ch.account) @@ -147,6 +170,9 @@ func (ch createObjectChange) dirtied() *common.Address { func (ch resetObjectChange) revert(s *DB) { s.setStateObject(ch.prev) + if !ch.prevdestruct { + delete(s.stateObjectsDestruct, ch.prev.address) + } } func (ch resetObjectChange) dirtied() *common.Address { @@ -198,14 +224,6 @@ func (ch codeChange) dirtied() *common.Address { return ch.account } -func (ch validatorWrapperChange) revert(s *DB) { - s.stateValidators[*(ch.address)] = ch.prev -} - -func (ch validatorWrapperChange) dirtied() *common.Address { - return ch.address -} - func (ch storageChange) revert(s *DB) { s.getStateObject(*ch.account).setState(ch.key, ch.prevalue) } @@ -214,6 +232,14 @@ func (ch storageChange) dirtied() *common.Address { return ch.account } +func (ch transientStorageChange) revert(s *DB) { + s.setTransientState(*ch.account, ch.key, ch.prevalue) +} + +func (ch transientStorageChange) dirtied() *common.Address { + return nil +} + func (ch refundChange) revert(s *DB) { s.refund = ch.prev } @@ -243,3 +269,28 @@ func (ch addPreimageChange) revert(s *DB) { func (ch addPreimageChange) dirtied() *common.Address { return nil } + +func (ch accessListAddAccountChange) revert(s *DB) { + /* + One important invariant here, is that whenever a (addr, slot) is added, if the + addr is not already present, the add causes two journal entries: + - one for the address, + - one for the (address,slot) + Therefore, when unrolling the change, we can always blindly delete the + (addr) at this point, since no storage adds can remain when come upon + a single (addr) change. + */ + s.accessList.DeleteAddress(*ch.address) +} + +func (ch accessListAddAccountChange) dirtied() *common.Address { + return nil +} + +func (ch accessListAddSlotChange) revert(s *DB) { + s.accessList.DeleteSlot(*ch.address, *ch.slot) +} + +func (ch accessListAddSlotChange) dirtied() *common.Address { + return nil +} diff --git a/core/state/managed_state_test.go b/core/state/managed_state_test.go index 2fc7ca5ad8..d1fbb906d5 100644 --- a/core/state/managed_state_test.go +++ b/core/state/managed_state_test.go @@ -27,7 +27,7 @@ import ( var addr = common.BytesToAddress([]byte("test")) func create() (*ManagedState, *account) { - statedb, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + statedb, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil) ms := ManageState(statedb) ms.DB.SetNonce(addr, 100) ms.accounts[addr] = newAccount(ms.DB.getStateObject(addr)) diff --git a/core/state/metrics.go b/core/state/metrics.go new file mode 100644 index 0000000000..e702ef3a81 --- /dev/null +++ b/core/state/metrics.go @@ -0,0 +1,30 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import "github.com/ethereum/go-ethereum/metrics" + +var ( + accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil) + storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil) + accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil) + storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil) + accountTrieUpdatedMeter = metrics.NewRegisteredMeter("state/update/accountnodes", nil) + storageTriesUpdatedMeter = metrics.NewRegisteredMeter("state/update/storagenodes", nil) + accountTrieDeletedMeter = metrics.NewRegisteredMeter("state/delete/accountnodes", nil) + storageTriesDeletedMeter = metrics.NewRegisteredMeter("state/delete/storagenodes", nil) +) diff --git a/core/state/prefeth.go b/core/state/prefeth.go index 50094fa63d..4d3b317bb1 100644 --- a/core/state/prefeth.go +++ b/core/state/prefeth.go @@ -8,6 +8,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/harmony-one/harmony/internal/utils" @@ -15,7 +16,7 @@ import ( type prefetchJob struct { accountAddr []byte - account *Account + account *types.StateAccount start, end []byte } @@ -91,7 +92,7 @@ func (s *DB) prefetchWorker(job *prefetchJob, jobs chan *prefetchJob) { } // build account data from main trie tree - var data Account + var data types.StateAccount if err := rlp.DecodeBytes(it.Value, &data); err != nil { panic(err) } @@ -103,7 +104,8 @@ func (s *DB) prefetchWorker(job *prefetchJob, jobs chan *prefetchJob) { } // build account trie tree - storageIt := trie.NewIterator(obj.getTrie(s.db).NodeIterator(nil)) + tr, _ := obj.getTrie(s.db) + storageIt := trie.NewIterator(tr.NodeIterator(nil)) storageJob := &prefetchJob{ accountAddr: addrBytes, account: &data, @@ -115,7 +117,8 @@ func (s *DB) prefetchWorker(job *prefetchJob, jobs chan *prefetchJob) { } else { // scan main trie tree obj := newObject(s, common.BytesToAddress(job.accountAddr), *job.account) - storageIt := trie.NewIterator(obj.getTrie(s.db).NodeIterator(job.start)) + tr, _ := obj.getTrie(s.db) + storageIt := trie.NewIterator(tr.NodeIterator(job.start)) // fetch data s.prefetchAccountStorage(jobs, job, storageIt) diff --git a/core/state/state_object.go b/core/state/state_object.go index 0d4fd7a800..a235b18f73 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -23,12 +23,12 @@ import ( "math/big" "time" - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" - + "github.com/ethereum/go-ethereum/trie" "github.com/harmony-one/harmony/staking" ) @@ -54,7 +54,7 @@ func (s Storage) String() (str string) { // Copy ... func (s Storage) Copy() Storage { - cpy := make(Storage) + cpy := make(Storage, len(s)) for key, value := range s { cpy[key] = value } @@ -67,18 +67,18 @@ func (s Storage) Copy() Storage { // The usage pattern is as follows: // First you need to obtain a state object. // Account values can be accessed and modified through the object. -// Finally, call CommitTrie to write the modified storage trie into a database. +// Finally, call commitTrie to write the modified storage trie into a database. type Object struct { address common.Address addrHash common.Hash // hash of ethereum address of the account - data Account + data types.StateAccount db *DB // DB error. // State objects are used by the consensus core and VM which are // unable to deal with database-level errors. Any error that occurs // during a database read is memoized here and will eventually be returned - // by StateDB.Commit. + // by DB.Commit. dbErr error // Write caches. @@ -113,15 +113,15 @@ type Account struct { } // newObject creates a state object. -func newObject(db *DB, address common.Address, data Account) *Object { +func newObject(db *DB, address common.Address, data types.StateAccount) *Object { if data.Balance == nil { data.Balance = new(big.Int) } if data.CodeHash == nil { - data.CodeHash = emptyCodeHash + data.CodeHash = types.EmptyCodeHash.Bytes() } if data.Root == (common.Hash{}) { - data.Root = emptyRoot + data.Root = types.EmptyRootHash } return &Object{ db: db, @@ -136,7 +136,7 @@ func newObject(db *DB, address common.Address, data Account) *Object { // EncodeRLP implements rlp.Encoder. func (s *Object) EncodeRLP(w io.Writer) error { - return rlp.Encode(w, s.data) + return rlp.Encode(w, &s.data) } // setError remembers the first non-nil error it is called with. @@ -161,16 +161,27 @@ func (s *Object) touch() { } } -func (s *Object) getTrie(db Database) Trie { +// getTrie returns the associated storage trie. The trie will be opened +// if it's not loaded previously. An error will be returned if trie can't +// be loaded. +func (s *Object) getTrie(db Database) (Trie, error) { if s.trie == nil { - var err error - s.trie, err = db.OpenStorageTrie(s.addrHash, s.data.Root) - if err != nil { - s.trie, _ = db.OpenStorageTrie(s.addrHash, common.Hash{}) - s.setError(fmt.Errorf("can't create storage trie: %v", err)) + // Try fetching from prefetcher first + // We don't prefetch empty tries + if s.data.Root != types.EmptyRootHash && s.db.prefetcher != nil { + // When the miner is creating the pending state, there is no + // prefetcher + s.trie = s.db.prefetcher.trie(s.addrHash, s.data.Root) + } + if s.trie == nil { + tr, err := db.OpenStorageTrie(s.db.originalRoot, s.addrHash, s.data.Root) + if err != nil { + return nil, err + } + s.trie = tr } } - return s.trie + return s.trie, nil } // GetState retrieves a value from the account storage trie. @@ -201,16 +212,44 @@ func (s *Object) GetCommittedState(db Database, key common.Hash) common.Hash { if value, cached := s.originStorage[key]; cached { return value } - // Track the amount of time wasted on reading the storage trie - if metrics.EnabledExpensive { - defer func(start time.Time) { s.db.StorageReads += time.Since(start) }(time.Now()) - } - // Otherwise load the value from the database - enc, err := s.getTrie(db).TryGet(key[:]) - if err != nil { - s.setError(err) + // If the object was destructed in *this* block (and potentially resurrected), + // the storage has been cleared out, and we should *not* consult the previous + // database about any storage values. The only possible alternatives are: + // 1) resurrect happened, and new slot values were set -- those should + // have been handles via pendingStorage above. + // 2) we don't have new values, and can deliver empty response back + if _, destructed := s.db.stateObjectsDestruct[s.address]; destructed { return common.Hash{} } + // If no live objects are available, attempt to use snapshots + var ( + enc []byte + err error + ) + if s.db.snap != nil { + start := time.Now() + enc, err = s.db.snap.Storage(s.addrHash, crypto.Keccak256Hash(key.Bytes())) + if metrics.EnabledExpensive { + s.db.SnapshotStorageReads += time.Since(start) + } + } + // If the snapshot is unavailable or reading from it fails, load from the database. + if s.db.snap == nil || err != nil { + start := time.Now() + tr, err := s.getTrie(db) + if err != nil { + s.setError(err) + return common.Hash{} + } + enc, err = tr.TryGet(key.Bytes()) + if metrics.EnabledExpensive { + s.db.StorageReads += time.Since(start) + } + if err != nil { + s.setError(err) + return common.Hash{} + } + } var value common.Hash if len(enc) > 0 { _, content, _, err := rlp.Split(enc) @@ -268,9 +307,16 @@ func (s *Object) setState(key, value common.Hash) { // finalise moves all dirty storage slots into the pending area to be hashed or // committed later. It is invoked at the end of every transaction. -func (s *Object) finalise() { +func (s *Object) finalise(prefetch bool) { + slotsToPrefetch := make([][]byte, 0, len(s.dirtyStorage)) for key, value := range s.dirtyStorage { s.pendingStorage[key] = value + if value != s.originStorage[key] { + slotsToPrefetch = append(slotsToPrefetch, common.CopyBytes(key[:])) // Copy needed for closure + } + } + if s.db.prefetcher != nil && prefetch && len(slotsToPrefetch) > 0 && s.data.Root != types.EmptyRootHash { + s.db.prefetcher.prefetch(s.addrHash, s.data.Root, slotsToPrefetch) } if len(s.dirtyStorage) > 0 { s.dirtyStorage = make(Storage) @@ -278,16 +324,30 @@ func (s *Object) finalise() { } // updateTrie writes cached storage modifications into the object's storage trie. -func (s *Object) updateTrie(db Database) Trie { +// It will return nil if the trie has not been loaded and no changes have been +// made. An error will be returned if the trie can't be loaded/updated correctly. +func (s *Object) updateTrie(db Database) (Trie, error) { // Make sure all dirty slots are finalized into the pending storage area - s.finalise() - - // Track the amount of time wasted on updating the storge trie + s.finalise(false) // Don't prefetch anymore, pull directly if need be + if len(s.pendingStorage) == 0 { + return s.trie, nil + } + // Track the amount of time wasted on updating the storage trie if metrics.EnabledExpensive { defer func(start time.Time) { s.db.StorageUpdates += time.Since(start) }(time.Now()) } + // The snapshot storage map for the object + var ( + storage map[common.Hash][]byte + hasher = s.db.hasher + ) + tr, err := s.getTrie(db) + if err != nil { + s.setError(err) + return nil, err + } // Insert all the pending updates into the trie - tr := s.getTrie(db) + usedStorage := make([][]byte, 0, len(s.pendingStorage)) for key, value := range s.pendingStorage { // Skip noop changes, persist actual changes if value == s.originStorage[key] { @@ -295,65 +355,101 @@ func (s *Object) updateTrie(db Database) Trie { } s.originStorage[key] = value + var v []byte if (value == common.Hash{}) { - s.setError(tr.TryDelete(key[:])) - continue + if err := tr.TryDelete(key[:]); err != nil { + s.setError(err) + return nil, err + } + s.db.StorageDeleted += 1 + } else { + // Encoding []byte cannot fail, ok to ignore the error. + v, _ = rlp.EncodeToBytes(common.TrimLeftZeroes(value[:])) + if err := tr.TryUpdate(key[:], v); err != nil { + s.setError(err) + return nil, err + } + s.db.StorageUpdated += 1 + } + // If state snapshotting is active, cache the data til commit + if s.db.snap != nil { + if storage == nil { + // Retrieve the old storage map, if available, create a new one otherwise + if storage = s.db.snapStorage[s.addrHash]; storage == nil { + storage = make(map[common.Hash][]byte) + s.db.snapStorage[s.addrHash] = storage + } + } + storage[crypto.HashData(hasher, key[:])] = v // v will be nil if it's deleted } - // Encoding []byte cannot fail, ok to ignore the error. - v, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(value[:])) - s.setError(tr.TryUpdate(key[:], v)) + usedStorage = append(usedStorage, common.CopyBytes(key[:])) // Copy needed for closure + } + if s.db.prefetcher != nil { + s.db.prefetcher.used(s.addrHash, s.data.Root, usedStorage) } if len(s.pendingStorage) > 0 { s.pendingStorage = make(Storage) } - return tr + return tr, nil } -// UpdateRoot sets the trie root to the current root hash of +// UpdateRoot sets the trie root to the current root hash of. An error +// will be returned if trie root hash is not computed correctly. func (s *Object) updateRoot(db Database) { - s.updateTrie(db) - - // Track the amount of time wasted on hashing the storge trie + tr, err := s.updateTrie(db) + if err != nil { + s.setError(fmt.Errorf("updateRoot (%x) error: %w", s.address, err)) + return + } + // If nothing changed, don't bother with hashing anything + if tr == nil { + return + } + // Track the amount of time wasted on hashing the storage trie if metrics.EnabledExpensive { defer func(start time.Time) { s.db.StorageHashes += time.Since(start) }(time.Now()) } - s.data.Root = s.trie.Hash() + s.data.Root = tr.Hash() } -// CommitTrie the storage trie of the object to db. -// This updates the trie root. -func (s *Object) CommitTrie(db Database) error { - s.updateTrie(db) +// commitTrie submits the storage changes into the storage trie and re-computes +// the root. Besides, all trie changes will be collected in a nodeset and returned. +func (s *Object) commitTrie(db Database) (*trie.NodeSet, error) { + tr, err := s.updateTrie(db) + if err != nil { + return nil, err + } if s.dbErr != nil { - return s.dbErr + return nil, s.dbErr + } + // If nothing changed, don't bother with committing anything + if tr == nil { + return nil, nil } - // Track the amount of time wasted on committing the storge trie + // Track the amount of time wasted on committing the storage trie if metrics.EnabledExpensive { defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now()) } - root, err := s.trie.Commit(nil) - if err == nil { - s.data.Root = root - } - return err + root, nodes := tr.Commit(false) + s.data.Root = root + return nodes, err } -// AddBalance removes amount from c's balance. +// AddBalance adds amount to s's balance. // It is used to add funds to the destination account of a transfer. func (s *Object) AddBalance(amount *big.Int) { - // EIP158: We must check emptiness for the objects such that the account + // EIP161: We must check emptiness for the objects such that the account // clearing (0,0,0 objects) can take effect. if amount.Sign() == 0 { if s.empty() { s.touch() } - return } s.SetBalance(new(big.Int).Add(s.Balance(), amount)) } -// SubBalance removes amount from c's balance. +// SubBalance removes amount from s's balance. // It is used to remove funds from the origin account of a transfer. func (s *Object) SubBalance(amount *big.Int) { if amount.Sign() == 0 { @@ -362,7 +458,6 @@ func (s *Object) SubBalance(amount *big.Int) { s.SetBalance(new(big.Int).Sub(s.Balance(), amount)) } -// SetBalance ... func (s *Object) SetBalance(amount *big.Int) { s.db.journal.append(balanceChange{ account: &s.address, @@ -379,18 +474,18 @@ func (s *Object) setBalance(amount *big.Int) { func (s *Object) ReturnGas(gas *big.Int) {} func (s *Object) deepCopy(db *DB) *Object { - stateObject := newObject(db, s.address, s.data) + Object := newObject(db, s.address, s.data) if s.trie != nil { - stateObject.trie = db.db.CopyTrie(s.trie) + Object.trie = db.db.CopyTrie(s.trie) } - stateObject.code = s.code - stateObject.dirtyStorage = s.dirtyStorage.Copy() - stateObject.originStorage = s.originStorage.Copy() - stateObject.pendingStorage = s.pendingStorage.Copy() - stateObject.suicided = s.suicided - stateObject.dirtyCode = s.dirtyCode - stateObject.deleted = s.deleted - return stateObject + Object.code = s.code + Object.dirtyStorage = s.dirtyStorage.Copy() + Object.originStorage = s.originStorage.Copy() + Object.pendingStorage = s.pendingStorage.Copy() + Object.suicided = s.suicided + Object.dirtyCode = s.dirtyCode + Object.deleted = s.deleted + return Object } // @@ -407,7 +502,7 @@ func (s *Object) Code(db Database) []byte { if s.code != nil { return s.code } - if bytes.Equal(s.CodeHash(), emptyCodeHash) { + if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) { return nil } code, err := db.ContractCode(s.addrHash, common.BytesToHash(s.CodeHash())) @@ -418,7 +513,23 @@ func (s *Object) Code(db Database) []byte { return code } -// SetCode ... +// CodeSize returns the size of the contract code associated with this object, +// or zero if none. This method is an almost mirror of Code, but uses a cache +// inside the database to avoid loading codes seen recently. +func (s *Object) CodeSize(db Database) int { + if s.code != nil { + return len(s.code) + } + if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) { + return 0 + } + size, err := db.ContractCodeSize(s.addrHash, common.BytesToHash(s.CodeHash())) + if err != nil { + s.setError(fmt.Errorf("can't load code size %x: %v", s.CodeHash(), err)) + } + return size +} + func (s *Object) SetCode(codeHash common.Hash, code []byte) { prevcode := s.Code(s.db.db) s.db.journal.append(codeChange{ @@ -435,7 +546,6 @@ func (s *Object) setCode(codeHash common.Hash, code []byte) { s.dirtyCode = true } -// SetNonce ... func (s *Object) SetNonce(nonce uint64) { s.db.journal.append(nonceChange{ account: &s.address, @@ -448,26 +558,23 @@ func (s *Object) setNonce(nonce uint64) { s.data.Nonce = nonce } -// CodeHash ... func (s *Object) CodeHash() []byte { return s.data.CodeHash } -// Balance ... func (s *Object) Balance() *big.Int { return s.data.Balance } -// Nonce ... func (s *Object) Nonce() uint64 { return s.data.Nonce } -// Value Never called, but must be present to allow stateObject to be used +// Value is never called, but must be present to allow Object to be used // as a vm.Account interface that also satisfies the vm.ContractRef // interface. Interfaces are awesome. func (s *Object) Value() *big.Int { - panic("Value on stateObject should never be called") + panic("Value on state object should never be called") } // IsValidator checks whether it is a validator object diff --git a/core/state/state_object_test.go b/core/state/state_object_test.go new file mode 100644 index 0000000000..42fd778025 --- /dev/null +++ b/core/state/state_object_test.go @@ -0,0 +1,46 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "bytes" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +func BenchmarkCutOriginal(b *testing.B) { + value := common.HexToHash("0x01") + for i := 0; i < b.N; i++ { + bytes.TrimLeft(value[:], "\x00") + } +} + +func BenchmarkCutsetterFn(b *testing.B) { + value := common.HexToHash("0x01") + cutSetFn := func(r rune) bool { return r == 0 } + for i := 0; i < b.N; i++ { + bytes.TrimLeftFunc(value[:], cutSetFn) + } +} + +func BenchmarkCutCustomTrim(b *testing.B) { + value := common.HexToHash("0x01") + for i := 0; i < b.N; i++ { + common.TrimLeftZeroes(value[:]) + } +} diff --git a/core/state/state_test.go b/core/state/state_test.go index 37eb128866..9209d3de29 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -22,9 +22,10 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie" + "github.com/harmony-one/harmony/core/rawdb" ) type stateTest struct { @@ -34,12 +35,14 @@ type stateTest struct { func newStateTest() *stateTest { db := rawdb.NewMemoryDatabase() - sdb, _ := New(common.Hash{}, NewDatabase(db)) + sdb, _ := New(common.Hash{}, NewDatabase(db), nil) return &stateTest{db: db, state: sdb} } func TestDump(t *testing.T) { - s := newStateTest() + db := rawdb.NewMemoryDatabase() + sdb, _ := New(common.Hash{}, NewDatabaseWithConfig(db, &trie.Config{Preimages: true}), nil) + s := &stateTest{db: db, state: sdb} // generate a few entries obj1 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x01})) @@ -147,7 +150,7 @@ func TestSnapshotEmpty(t *testing.T) { } func TestSnapshot2(t *testing.T) { - state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil) stateobjaddr0 := common.BytesToAddress([]byte("so0")) stateobjaddr1 := common.BytesToAddress([]byte("so1")) @@ -169,7 +172,7 @@ func TestSnapshot2(t *testing.T) { state.setStateObject(so0) root, _ := state.Commit(false) - state.Reset(root) + state, _ = New(root, state.db, state.snaps) // and one with deleted == true so1 := state.getStateObject(stateobjaddr1) diff --git a/core/state/statedb.go b/core/state/statedb.go index 0a7cd3b564..eff2a37c2a 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -23,14 +23,16 @@ import ( "sort" "time" - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/core/rawdb" + + types2 "github.com/harmony-one/harmony/core/types" common2 "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/numeric" @@ -65,20 +67,32 @@ func (n *proofList) Delete(key []byte) error { panic("not supported") } -// DB within the ethereum protocol are used to store anything +// DB structs within the ethereum protocol are used to store anything // within the merkle trie. StateDBs take care of caching and storing // nested states. It's the general query interface to retrieve: // * Contracts // * Accounts type DB struct { - db Database - trie Trie + db Database + prefetcher *triePrefetcher + trie Trie + hasher crypto.KeccakState + + // originalRoot is the pre-state root, before any changes were made. + // It will be updated when the Commit is called. + originalRoot common.Hash + + snaps *snapshot.Tree + snap snapshot.Snapshot + snapAccounts map[common.Hash][]byte + snapStorage map[common.Hash]map[common.Hash][]byte // This map holds 'live' objects, which will get modified while processing a state transition. - stateObjects map[common.Address]*Object - stateObjectsPending map[common.Address]struct{} // State objects finalized but not yet written to the trie - stateObjectsDirty map[common.Address]struct{} - stateValidators map[common.Address]*stk.ValidatorWrapper + stateObjects map[common.Address]*Object + stateObjectsPending map[common.Address]struct{} // State objects finalized but not yet written to the trie + stateObjectsDirty map[common.Address]struct{} // State objects modified in the current execution + stateObjectsDestruct map[common.Address]struct{} // State objects destructed in the block + stateValidators map[common.Address]*stk.ValidatorWrapper // DB error. // State objects are used by the consensus core and VM which are @@ -93,45 +107,95 @@ type DB struct { thash, bhash common.Hash // thash means hmy tx hash ethTxHash common.Hash // ethTxHash is eth tx hash, use by tracer txIndex int - logs map[common.Hash][]*types.Log + logs map[common.Hash][]*types2.Log logSize uint preimages map[common.Hash][]byte + // Per-transaction access list + accessList *accessList + + // Transient storage + transientStorage transientStorage + // Journal of state modifications. This is the backbone of // Snapshot and RevertToSnapshot. journal *journal validRevisions []revision - nextRevisionID int + nextRevisionId int // Measurements gathered during execution for debugging purposes - AccountReads time.Duration - AccountHashes time.Duration - AccountUpdates time.Duration - AccountCommits time.Duration - StorageReads time.Duration - StorageHashes time.Duration - StorageUpdates time.Duration - StorageCommits time.Duration + AccountReads time.Duration + AccountHashes time.Duration + AccountUpdates time.Duration + AccountCommits time.Duration + StorageReads time.Duration + StorageHashes time.Duration + StorageUpdates time.Duration + StorageCommits time.Duration + SnapshotAccountReads time.Duration + SnapshotStorageReads time.Duration + SnapshotCommits time.Duration + TrieDBCommits time.Duration + + AccountUpdated int + StorageUpdated int + AccountDeleted int + StorageDeleted int } // New creates a new state from a given trie. -func New(root common.Hash, db Database) (*DB, error) { +func New(root common.Hash, db Database, snaps *snapshot.Tree) (*DB, error) { tr, err := db.OpenTrie(root) if err != nil { return nil, err } - return &DB{ - db: db, - trie: tr, - stateObjects: make(map[common.Address]*Object), - stateObjectsPending: make(map[common.Address]struct{}), - stateObjectsDirty: make(map[common.Address]struct{}), - stateValidators: make(map[common.Address]*stk.ValidatorWrapper), - logs: make(map[common.Hash][]*types.Log), - preimages: make(map[common.Hash][]byte), - journal: newJournal(), - }, nil + sdb := &DB{ + db: db, + trie: tr, + originalRoot: root, + snaps: snaps, + stateObjects: make(map[common.Address]*Object), + stateObjectsPending: make(map[common.Address]struct{}), + stateObjectsDirty: make(map[common.Address]struct{}), + stateObjectsDestruct: make(map[common.Address]struct{}), + stateValidators: make(map[common.Address]*stk.ValidatorWrapper), + logs: make(map[common.Hash][]*types2.Log), + preimages: make(map[common.Hash][]byte), + journal: newJournal(), + accessList: newAccessList(), + transientStorage: newTransientStorage(), + hasher: crypto.NewKeccakState(), + } + if sdb.snaps != nil { + if sdb.snap = sdb.snaps.Snapshot(root); sdb.snap != nil { + sdb.snapAccounts = make(map[common.Hash][]byte) + sdb.snapStorage = make(map[common.Hash]map[common.Hash][]byte) + } + } + return sdb, nil +} + +// StartPrefetcher initializes a new trie prefetcher to pull in nodes from the +// state trie concurrently while the state is mutated so that when we reach the +// commit phase, most of the needed data is already hot. +func (db *DB) StartPrefetcher(namespace string) { + if db.prefetcher != nil { + db.prefetcher.close() + db.prefetcher = nil + } + if db.snap != nil { + db.prefetcher = newTriePrefetcher(db.db, db.originalRoot, namespace) + } +} + +// StopPrefetcher terminates a running prefetcher and reports any leftover stats +// from the gathered metrics. +func (db *DB) StopPrefetcher() { + if db.prefetcher != nil { + db.prefetcher.close() + db.prefetcher = nil + } } // setError remembers the first non-nil error it is called with. @@ -161,15 +225,14 @@ func (db *DB) Reset(root common.Hash) error { db.bhash = common.Hash{} db.ethTxHash = common.Hash{} db.txIndex = 0 - db.logs = make(map[common.Hash][]*types.Log) + db.logs = make(map[common.Hash][]*types2.Log) db.logSize = 0 db.preimages = make(map[common.Hash][]byte) db.clearJournalAndRefund() return nil } -// AddLog ... -func (db *DB) AddLog(log *types.Log) { +func (db *DB) AddLog(log *types2.Log) { db.journal.append(addLogChange{txhash: db.thash}) log.TxHash = db.thash @@ -180,14 +243,19 @@ func (db *DB) AddLog(log *types.Log) { db.logSize++ } -// GetLogs ... -func (db *DB) GetLogs(hash common.Hash) []*types.Log { - return db.logs[hash] +// GetLogs returns the logs matching the specified transaction hash, and annotates +// them with the given blockNumber and blockHash. +func (db *DB) GetLogs(hash common.Hash, blockNumber uint64, blockHash common.Hash) []*types2.Log { + logs := db.logs[hash] + for _, l := range logs { + l.BlockNumber = blockNumber + l.BlockHash = blockHash + } + return logs } -// Logs ... -func (db *DB) Logs() []*types.Log { - var logs []*types.Log +func (db *DB) Logs() []*types2.Log { + var logs []*types2.Log for _, lgs := range db.logs { logs = append(logs, lgs...) } @@ -220,7 +288,7 @@ func (db *DB) AddRefund(gas uint64) { func (db *DB) SubRefund(gas uint64) { db.journal.append(refundChange{prev: db.refund}) if gas > db.refund { - panic("Refund counter below zero") + panic(fmt.Sprintf("Refund counter below zero (gas: %d > refund: %d)", gas, db.refund)) } db.refund -= gas } @@ -240,18 +308,17 @@ func (db *DB) Empty(addr common.Address) bool { // GetBalance retrieves the balance from the given address or 0 if object not found func (db *DB) GetBalance(addr common.Address) *big.Int { - stateObject := db.getStateObject(addr) - if stateObject != nil { - return stateObject.Balance() + Object := db.getStateObject(addr) + if Object != nil { + return Object.Balance() } return common.Big0 } -// GetNonce ... func (db *DB) GetNonce(addr common.Address) uint64 { - stateObject := db.getStateObject(addr) - if stateObject != nil { - return stateObject.Nonce() + Object := db.getStateObject(addr) + if Object != nil { + return Object.Nonce() } return 0 @@ -262,12 +329,12 @@ func (db *DB) TxIndex() int { return db.txIndex } -func (s *DB) TxHash() common.Hash { - return s.thash +func (db *DB) TxHash() common.Hash { + return db.thash } -func (s *DB) TxHashETH() common.Hash { - return s.ethTxHash +func (db *DB) TxHashETH() common.Hash { + return db.ethTxHash } // BlockHash returns the current block hash set by Prepare. @@ -275,74 +342,73 @@ func (db *DB) BlockHash() common.Hash { return db.bhash } -// GetCode ... func (db *DB) GetCode(addr common.Address) []byte { - stateObject := db.getStateObject(addr) - if stateObject != nil { - return stateObject.Code(db.db) + Object := db.getStateObject(addr) + if Object != nil { + return Object.Code(db.db) } return nil } -// GetCodeSize ... func (db *DB) GetCodeSize(addr common.Address) int { - stateObject := db.getStateObject(addr) - if stateObject == nil { - return 0 - } - if stateObject.code != nil { - return len(stateObject.code) - } - size, err := db.db.ContractCodeSize( - stateObject.addrHash, common.BytesToHash(stateObject.CodeHash()), - ) - if err != nil { - db.setError(err) + Object := db.getStateObject(addr) + if Object != nil { + return Object.CodeSize(db.db) } - return size + return 0 } -// GetCodeHash ... func (db *DB) GetCodeHash(addr common.Address) common.Hash { - stateObject := db.getStateObject(addr) - if stateObject == nil { + Object := db.getStateObject(addr) + if Object == nil { return common.Hash{} } - return common.BytesToHash(stateObject.CodeHash()) + return common.BytesToHash(Object.CodeHash()) } // GetState retrieves a value from the given account's storage trie. func (db *DB) GetState(addr common.Address, hash common.Hash) common.Hash { - stateObject := db.getStateObject(addr) - if stateObject != nil { - return stateObject.GetState(db.db, hash) + Object := db.getStateObject(addr) + if Object != nil { + return Object.GetState(db.db, hash) } return common.Hash{} } -// GetProof returns the MerkleProof for a given Account -func (db *DB) GetProof(a common.Address) ([][]byte, error) { +// GetProof returns the Merkle proof for a given account. +func (db *DB) GetProof(addr common.Address) ([][]byte, error) { + return db.GetProofByHash(crypto.Keccak256Hash(addr.Bytes())) +} + +// GetProofByHash returns the Merkle proof for a given account. +func (db *DB) GetProofByHash(addrHash common.Hash) ([][]byte, error) { var proof proofList - err := db.trie.Prove(crypto.Keccak256(a.Bytes()), 0, &proof) - return [][]byte(proof), err + err := db.trie.Prove(addrHash[:], 0, &proof) + return proof, err } -// GetStorageProof returns the StorageProof for given key +// GetStorageProof returns the Merkle proof for given storage slot. func (db *DB) GetStorageProof(a common.Address, key common.Hash) ([][]byte, error) { - var proof proofList - trie := db.StorageTrie(a) + trie, err := db.StorageTrie(a) + if err != nil { + return nil, err + } if trie == nil { - return proof, errors.New("storage trie for requested address does not exist") + return nil, errors.New("storage trie for requested address does not exist") + } + var proof proofList + err = trie.Prove(crypto.Keccak256(key.Bytes()), 0, &proof) + if err != nil { + return nil, err } - err := trie.Prove(crypto.Keccak256(key.Bytes()), 0, &proof) - return [][]byte(proof), err + return proof, nil } // GetCommittedState retrieves a value from the given account's committed storage trie. func (db *DB) GetCommittedState(addr common.Address, hash common.Hash) common.Hash { - stateObject := db.getStateObject(addr) - if stateObject != nil { - return stateObject.GetCommittedState(db.db, hash) + Object := db.getStateObject(addr) + if Object != nil { + return Object.GetCommittedState(db.db, hash) } return common.Hash{} } @@ -352,22 +418,25 @@ func (db *DB) Database() Database { return db.db } -// StorageTrie returns the storage trie of an account. -// The return value is a copy and is nil for non-existent accounts. -func (db *DB) StorageTrie(addr common.Address) Trie { - stateObject := db.getStateObject(addr) - if stateObject == nil { - return nil +// StorageTrie returns the storage trie of an account. The return value is a copy +// and is nil for non-existent accounts. An error will be returned if storage trie +// is existent but can't be loaded correctly. +func (db *DB) StorageTrie(addr common.Address) (Trie, error) { + Object := db.getStateObject(addr) + if Object == nil { + return nil, nil } - cpy := stateObject.deepCopy(db) - return cpy.updateTrie(db.db) + cpy := Object.deepCopy(db) + if _, err := cpy.updateTrie(db.db); err != nil { + return nil, err + } + return cpy.getTrie(db.db) } -// HasSuicided ... func (db *DB) HasSuicided(addr common.Address) bool { - stateObject := db.getStateObject(addr) - if stateObject != nil { - return stateObject.suicided + Object := db.getStateObject(addr) + if Object != nil { + return Object.suicided } return false } @@ -378,49 +447,60 @@ func (db *DB) HasSuicided(addr common.Address) bool { // AddBalance adds amount to the account associated with addr. func (db *DB) AddBalance(addr common.Address, amount *big.Int) { - stateObject := db.GetOrNewStateObject(addr) - if stateObject != nil { - stateObject.AddBalance(amount) + Object := db.GetOrNewStateObject(addr) + if Object != nil { + Object.AddBalance(amount) } } // SubBalance subtracts amount from the account associated with addr. func (db *DB) SubBalance(addr common.Address, amount *big.Int) { - stateObject := db.GetOrNewStateObject(addr) - if stateObject != nil { - stateObject.SubBalance(amount) + Object := db.GetOrNewStateObject(addr) + if Object != nil { + Object.SubBalance(amount) } } -// SetBalance ... func (db *DB) SetBalance(addr common.Address, amount *big.Int) { - stateObject := db.GetOrNewStateObject(addr) - if stateObject != nil { - stateObject.SetBalance(amount) + Object := db.GetOrNewStateObject(addr) + if Object != nil { + Object.SetBalance(amount) } } -// SetNonce ... func (db *DB) SetNonce(addr common.Address, nonce uint64) { - stateObject := db.GetOrNewStateObject(addr) - if stateObject != nil { - stateObject.SetNonce(nonce) + Object := db.GetOrNewStateObject(addr) + if Object != nil { + Object.SetNonce(nonce) } } -// SetCode ... func (db *DB) SetCode(addr common.Address, code []byte) { - stateObject := db.GetOrNewStateObject(addr) - if stateObject != nil { - stateObject.SetCode(crypto.Keccak256Hash(code), code) + Object := db.GetOrNewStateObject(addr) + if Object != nil { + Object.SetCode(crypto.Keccak256Hash(code), code) } } -// SetState ... func (db *DB) SetState(addr common.Address, key, value common.Hash) { - stateObject := db.GetOrNewStateObject(addr) - if stateObject != nil { - stateObject.SetState(db.db, key, value) + Object := db.GetOrNewStateObject(addr) + if Object != nil { + Object.SetState(db.db, key, value) + } +} + +// SetStorage replaces the entire storage for the specified account with given +// storage. This function should only be used for debugging. +func (db *DB) SetStorage(addr common.Address, storage map[common.Hash]common.Hash) { + // SetStorage needs to wipe existing storage. We achieve this by pretending + // that the account self-destructed earlier in this block, by flagging + // it in stateObjectsDestruct. The effect of doing so is that storage lookups + // will not hit disk, since it is assumed that the disk-data is belonging + // to a previous incarnation of the object. + db.stateObjectsDestruct[addr] = struct{}{} + Object := db.GetOrNewStateObject(addr) + for k, v := range storage { + Object.SetState(db.db, k, v) } } @@ -430,21 +510,50 @@ func (db *DB) SetState(addr common.Address, key, value common.Hash) { // The account's state object is still available until the state is committed, // getStateObject will return a non-nil account after Suicide. func (db *DB) Suicide(addr common.Address) bool { - stateObject := db.getStateObject(addr) - if stateObject == nil { + Object := db.getStateObject(addr) + if Object == nil { return false } db.journal.append(suicideChange{ account: &addr, - prev: stateObject.suicided, - prevbalance: new(big.Int).Set(stateObject.Balance()), + prev: Object.suicided, + prevbalance: new(big.Int).Set(Object.Balance()), }) - stateObject.markSuicided() - stateObject.data.Balance = new(big.Int) + Object.markSuicided() + Object.data.Balance = new(big.Int) return true } +// SetTransientState sets transient storage for a given account. It +// adds the change to the journal so that it can be rolled back +// to its previous value if there is a revert. +func (db *DB) SetTransientState(addr common.Address, key, value common.Hash) { + prev := db.GetTransientState(addr, key) + if prev == value { + return + } + + db.journal.append(transientStorageChange{ + account: &addr, + key: key, + prevalue: prev, + }) + + db.setTransientState(addr, key, value) +} + +// setTransientState is a lower level setter for transient storage. It +// is called during a revert to prevent modifications to the journal. +func (db *DB) setTransientState(addr common.Address, key, value common.Hash) { + db.transientStorage.Set(addr, key, value) +} + +// GetTransientState gets transient storage for a given account. +func (db *DB) GetTransientState(addr common.Address, key common.Hash) common.Hash { + return db.transientStorage.Get(addr, key) +} + // // Setting, updating & deleting state object methods. // @@ -457,12 +566,17 @@ func (db *DB) updateStateObject(obj *Object) { } // Encode the account and update the account trie addr := obj.Address() + if err := db.trie.TryUpdateAccount(addr, &obj.data); err != nil { + db.setError(fmt.Errorf("updateStateObject (%x) error: %v", addr[:], err)) + } - data, err := rlp.EncodeToBytes(obj) - if err != nil { - panic(fmt.Errorf("can't encode object at %x: %v", addr[:], err)) + // If state snapshotting is active, cache the data til commit. Note, this + // update mechanism is not symmetric to the deletion, because whereas it is + // enough to track account updates at commit time, deletions need tracking + // at transaction boundary level to ensure we capture state clearing. + if db.snap != nil { + db.snapAccounts[obj.addrHash] = snapshot.SlimAccountRLP(obj.data.Nonce, obj.data.Balance, obj.data.Root, obj.data.CodeHash) } - db.setError(db.trie.TryUpdate(addr[:], data)) } // deleteStateObject removes the given object from the state trie. @@ -473,7 +587,9 @@ func (db *DB) deleteStateObject(obj *Object) { } // Delete the account from the trie addr := obj.Address() - db.setError(db.trie.TryDelete(addr[:])) + if err := db.trie.TryDeleteAccount(addr); err != nil { + db.setError(fmt.Errorf("deleteStateObject (%x) error: %v", addr[:], err)) + } } // getStateObject retrieves a state object given by the address, returning nil if @@ -495,23 +611,50 @@ func (db *DB) getDeletedStateObject(addr common.Address) *Object { if obj := db.stateObjects[addr]; obj != nil { return obj } - // Track the amount of time wasted on loading the object from the database - if metrics.EnabledExpensive { - defer func(start time.Time) { db.AccountReads += time.Since(start) }(time.Now()) - } - // Load the object from the database - enc, err := db.trie.TryGet(addr[:]) - if len(enc) == 0 { - db.setError(err) - return nil + // If no live objects are available, attempt to use snapshots + var data *types.StateAccount + if db.snap != nil { + start := time.Now() + acc, err := db.snap.Account(crypto.HashData(db.hasher, addr.Bytes())) + if metrics.EnabledExpensive { + db.SnapshotAccountReads += time.Since(start) + } + if err == nil { + if acc == nil { + return nil + } + data = &types.StateAccount{ + Nonce: acc.Nonce, + Balance: acc.Balance, + CodeHash: acc.CodeHash, + Root: common.BytesToHash(acc.Root), + } + if len(data.CodeHash) == 0 { + data.CodeHash = types.EmptyCodeHash.Bytes() + } + if data.Root == (common.Hash{}) { + data.Root = types.EmptyRootHash + } + } } - var data Account - if err := rlp.DecodeBytes(enc, &data); err != nil { - log.Error("Failed to decode state object", "addr", addr, "err", err) - return nil + // If snapshot unavailable or reading from it failed, load from the database + if data == nil { + start := time.Now() + var err error + data, err = db.trie.TryGetAccount(addr) + if metrics.EnabledExpensive { + db.AccountReads += time.Since(start) + } + if err != nil { + db.setError(fmt.Errorf("getDeleteStateObject (%x) error: %w", addr.Bytes(), err)) + return nil + } + if data == nil { + return nil + } } // Insert into the live set - obj := newObject(db, addr, data) + obj := newObject(db, addr, *data) db.setStateObject(obj) return obj } @@ -522,11 +665,11 @@ func (db *DB) setStateObject(object *Object) { // GetOrNewStateObject retrieves a state object or create a new state object if nil. func (db *DB) GetOrNewStateObject(addr common.Address) *Object { - stateObject := db.getStateObject(addr) - if stateObject == nil { - stateObject, _ = db.createObject(addr) + Object := db.getStateObject(addr) + if Object == nil { + Object, _ = db.createObject(addr) } - return stateObject + return Object } // createObject creates a new state object. If there is an existing account with @@ -534,12 +677,18 @@ func (db *DB) GetOrNewStateObject(addr common.Address) *Object { func (db *DB) createObject(addr common.Address) (newobj, prev *Object) { prev = db.getDeletedStateObject(addr) // Note, prev might have been deleted, we need that! - newobj = newObject(db, addr, Account{}) - newobj.setNonce(0) // sets the object to dirty + var prevdestruct bool + if prev != nil { + _, prevdestruct = db.stateObjectsDestruct[prev.address] + if !prevdestruct { + db.stateObjectsDestruct[prev.address] = struct{}{} + } + } + newobj = newObject(db, addr, types.StateAccount{}) if prev == nil { db.journal.append(createObjectChange{account: &addr}) } else { - db.journal.append(resetObjectChange{prev: prev}) + db.journal.append(resetObjectChange{prev: prev, prevdestruct: prevdestruct}) } db.setStateObject(newobj) if prev != nil && !prev.deleted { @@ -565,13 +714,16 @@ func (db *DB) CreateAccount(addr common.Address) { } } -// ForEachStorage ... func (db *DB) ForEachStorage(addr common.Address, cb func(key, value common.Hash) bool) error { so := db.getStateObject(addr) if so == nil { return nil } - it := trie.NewIterator(so.getTrie(db.db).NodeIterator(nil)) + tr, err := so.getTrie(db.db) + if err != nil { + return err + } + it := trie.NewIterator(tr.NodeIterator(nil)) for it.Next() { key := common.BytesToHash(db.trie.GetKey(it.Key)) @@ -600,17 +752,20 @@ func (db *DB) ForEachStorage(addr common.Address, cb func(key, value common.Hash func (db *DB) Copy() *DB { // Copy all the basic fields, initialize the memory ones state := &DB{ - db: db.db, - trie: db.db.CopyTrie(db.trie), - stateObjects: make(map[common.Address]*Object, len(db.journal.dirties)), - stateObjectsPending: make(map[common.Address]struct{}, len(db.stateObjectsPending)), - stateObjectsDirty: make(map[common.Address]struct{}, len(db.journal.dirties)), - stateValidators: make(map[common.Address]*stk.ValidatorWrapper), - refund: db.refund, - logs: make(map[common.Hash][]*types.Log, len(db.logs)), - logSize: db.logSize, - preimages: make(map[common.Hash][]byte), - journal: newJournal(), + db: db.db, + trie: db.db.CopyTrie(db.trie), + originalRoot: db.originalRoot, + stateObjects: make(map[common.Address]*Object, len(db.journal.dirties)), + stateObjectsPending: make(map[common.Address]struct{}, len(db.stateObjectsPending)), + stateObjectsDirty: make(map[common.Address]struct{}, len(db.journal.dirties)), + stateObjectsDestruct: make(map[common.Address]struct{}, len(db.stateObjectsDestruct)), + stateValidators: make(map[common.Address]*stk.ValidatorWrapper), + refund: db.refund, + logs: make(map[common.Hash][]*types2.Log, len(db.logs)), + logSize: db.logSize, + preimages: make(map[common.Hash][]byte, len(db.preimages)), + journal: newJournal(), + hasher: crypto.NewKeccakState(), } // Copy the dirty states, logs, and preimages for addr := range db.journal.dirties { @@ -620,7 +775,7 @@ func (db *DB) Copy() *DB { // nil if object, exist := db.stateObjects[addr]; exist { // Even though the original object is dirty, we are not copying the journal, - // so we need to make sure that anyside effect the journal would have caused + // so we need to make sure that any side-effect the journal would have caused // during a commit (or similar op) is already applied to the copy. state.stateObjects[addr] = object.deepCopy(state) @@ -628,9 +783,10 @@ func (db *DB) Copy() *DB { state.stateObjectsPending[addr] = struct{}{} // Mark the copy pending to force external (account) commits } } - // Above, we don't copy the actual journal. This means that if the copy is copied, the - // loop above will be a no-op, since the copy's journal is empty. - // Thus, here we iterate over stateObjects, to enable copies of copies + // Above, we don't copy the actual journal. This means that if the copy + // is copied, the loop above will be a no-op, since the copy's journal + // is empty. Thus, here we iterate over stateObjects, to enable copies + // of copies. for addr := range db.stateObjectsPending { if _, exist := state.stateObjects[addr]; !exist { state.stateObjects[addr] = db.stateObjects[addr].deepCopy(state) @@ -643,14 +799,18 @@ func (db *DB) Copy() *DB { } state.stateObjectsDirty[addr] = struct{}{} } + // Deep copy the destruction flag. + for addr := range db.stateObjectsDestruct { + state.stateObjectsDestruct[addr] = struct{}{} + } for addr, wrapper := range db.stateValidators { copied := staketest.CopyValidatorWrapper(*wrapper) state.stateValidators[addr] = &copied } for hash, logs := range db.logs { - cpy := make([]*types.Log, len(logs)) + cpy := make([]*types2.Log, len(logs)) for i, l := range logs { - cpy[i] = new(types.Log) + cpy[i] = new(types2.Log) *cpy[i] = *l } state.logs[hash] = cpy @@ -658,13 +818,50 @@ func (db *DB) Copy() *DB { for hash, preimage := range db.preimages { state.preimages[hash] = preimage } + // Do we need to copy the access list and transient storage? + // In practice: No. At the start of a transaction, these two lists are empty. + // In practice, we only ever copy state _between_ transactions/blocks, never + // in the middle of a transaction. However, it doesn't cost us much to copy + // empty lists, so we do it anyway to not blow up if we ever decide copy them + // in the middle of a transaction. + state.accessList = db.accessList.Copy() + state.transientStorage = db.transientStorage.Copy() + + // If there's a prefetcher running, make an inactive copy of it that can + // only access data but does not actively preload (since the user will not + // know that they need to explicitly terminate an active copy). + if db.prefetcher != nil { + state.prefetcher = db.prefetcher.copy() + } + if db.snaps != nil { + // In order for the miner to be able to use and make additions + // to the snapshot tree, we need to copy that as well. + // Otherwise, any block mined by ourselves will cause gaps in the tree, + // and force the miner to operate trie-backed only + state.snaps = db.snaps + state.snap = db.snap + + // deep copy needed + state.snapAccounts = make(map[common.Hash][]byte) + for k, v := range db.snapAccounts { + state.snapAccounts[k] = v + } + state.snapStorage = make(map[common.Hash]map[common.Hash][]byte) + for k, v := range db.snapStorage { + temp := make(map[common.Hash][]byte) + for kk, vv := range v { + temp[kk] = vv + } + state.snapStorage[k] = temp + } + } return state } // Snapshot returns an identifier for the current revision of the state. func (db *DB) Snapshot() int { - id := db.nextRevisionID - db.nextRevisionID++ + id := db.nextRevisionId + db.nextRevisionId++ db.validRevisions = append(db.validRevisions, revision{id, db.journal.length()}) return id } @@ -690,15 +887,16 @@ func (db *DB) GetRefund() uint64 { return db.refund } -// Finalise finalises the state by removing the db destructed objects -// and clears the journal as well as the refunds. +// Finalise finalises the state by removing the destructed objects and clears +// the journal as well as the refunds. Finalise, however, will not push any updates +// into the tries just yet. Only IntermediateRoot or Commit will do that. func (db *DB) Finalise(deleteEmptyObjects bool) { // Commit validator changes in cache to stateObjects // TODO: remove validator cache after commit for addr, wrapper := range db.stateValidators { db.UpdateValidatorWrapper(addr, wrapper) } - + addressesToPrefetch := make([][]byte, 0, len(db.journal.dirties)) for addr := range db.journal.dirties { obj, exist := db.stateObjects[addr] if !exist { @@ -706,17 +904,38 @@ func (db *DB) Finalise(deleteEmptyObjects bool) { // That tx goes out of gas, and although the notion of 'touched' does not exist there, the // touch-event will still be recorded in the journal. Since ripeMD is a special snowflake, // it will persist in the journal even though the journal is reverted. In this special circumstance, - // it may exist in `s.journal.dirties` but not in `s.stateObjects`. + // it may exist in `db.journal.dirties` but not in `db.stateObjects`. // Thus, we can safely ignore it here continue } if obj.suicided || (deleteEmptyObjects && obj.empty()) { obj.deleted = true + + // We need to maintain account deletions explicitly (will remain + // set indefinitely). + db.stateObjectsDestruct[obj.address] = struct{}{} + + // If state snapshotting is active, also mark the destruction there. + // Note, we can't do this only at the end of a block because multiple + // transactions within the same block might self destruct and then + // resurrect an account; but the snapshotter needs both events. + if db.snap != nil { + delete(db.snapAccounts, obj.addrHash) // Clear out any previously updated account data (may be recreated via a resurrect) + delete(db.snapStorage, obj.addrHash) // Clear out any previously updated storage data (may be recreated via a resurrect) + } } else { - obj.finalise() + obj.finalise(true) // Prefetch slots in the background } db.stateObjectsPending[addr] = struct{}{} db.stateObjectsDirty[addr] = struct{}{} + + // At this point, also ship the address off to the precacher. The precacher + // will start loading tries, and when the change is eventually committed, + // the commit-phase will be a lot faster + addressesToPrefetch = append(addressesToPrefetch, common.CopyBytes(addr[:])) // Copy needed for closure + } + if db.prefetcher != nil && len(addressesToPrefetch) > 0 { + db.prefetcher.prefetch(common.Hash{}, db.originalRoot, addressesToPrefetch) } // Invalidate journal because reverting across transactions is not allowed. db.clearJournalAndRefund() @@ -729,14 +948,51 @@ func (db *DB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // Finalise all the dirty storage states and write them into the tries db.Finalise(deleteEmptyObjects) + // If there was a trie prefetcher operating, it gets aborted and irrevocably + // modified after we start retrieving tries. Remove it from the statedb after + // this round of use. + // + // This is weird pre-byzantium since the first tx runs with a prefetcher and + // the remainder without, but pre-byzantium even the initial prefetcher is + // useless, so no sleep lost. + prefetcher := db.prefetcher + if db.prefetcher != nil { + defer func() { + db.prefetcher.close() + db.prefetcher = nil + }() + } + // Although naively it makes sense to retrieve the account trie and then do + // the contract storage and account updates sequentially, that short circuits + // the account prefetcher. Instead, let's process all the storage updates + // first, giving the account prefetches just a few more milliseconds of time + // to pull useful data from disk. for addr := range db.stateObjectsPending { - obj := db.stateObjects[addr] - if obj.deleted { + if obj := db.stateObjects[addr]; !obj.deleted { + obj.updateRoot(db.db) + } + } + // Now we're about to start to write changes to the trie. The trie is so far + // _untouched_. We can check with the prefetcher, if it can give us a trie + // which has the same root, but also has some content loaded into it. + if prefetcher != nil { + if trie := prefetcher.trie(common.Hash{}, db.originalRoot); trie != nil { + db.trie = trie + } + } + usedAddrs := make([][]byte, 0, len(db.stateObjectsPending)) + for addr := range db.stateObjectsPending { + if obj := db.stateObjects[addr]; obj.deleted { db.deleteStateObject(obj) + db.AccountDeleted += 1 } else { - obj.updateRoot(db.db) db.updateStateObject(obj) + db.AccountUpdated += 1 } + usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure + } + if prefetcher != nil { + prefetcher.used(common.Hash{}, db.originalRoot, usedAddrs) } if len(db.stateObjectsPending) > 0 { db.stateObjectsPending = make(map[common.Address]struct{}) @@ -756,8 +1012,12 @@ func (db *DB) Prepare(thash, bhash common.Hash, ti int) { db.txIndex = ti } -func (db *DB) SetTxHashETH(ethTxHash common.Hash) { - db.ethTxHash = ethTxHash +// SetTxContext sets the current transaction hash and index which are +// used when the EVM emits new state logs. It should be invoked before +// transaction execution. +func (db *DB) SetTxContext(thash common.Hash, ti int) { + db.thash = thash + db.txIndex = ti } func (db *DB) clearJournalAndRefund() { @@ -765,49 +1025,191 @@ func (db *DB) clearJournalAndRefund() { db.journal = newJournal() db.refund = 0 } - db.validRevisions = db.validRevisions[:0] // Snapshots can be created without journal entires + db.validRevisions = db.validRevisions[:0] // Snapshots can be created without journal entries +} + +func (db *DB) SetTxHashETH(ethTxHash common.Hash) { + db.ethTxHash = ethTxHash } // Commit writes the state to the underlying in-memory trie database. -func (db *DB) Commit(deleteEmptyObjects bool) (root common.Hash, err error) { +func (db *DB) Commit(deleteEmptyObjects bool) (common.Hash, error) { + if db.dbErr != nil { + return common.Hash{}, fmt.Errorf("commit aborted due to earlier error: %v", db.dbErr) + } // Finalize any pending changes and merge everything into the tries db.IntermediateRoot(deleteEmptyObjects) // Commit objects to the trie, measuring the elapsed time + var ( + accountTrieNodesUpdated int + accountTrieNodesDeleted int + storageTrieNodesUpdated int + storageTrieNodesDeleted int + nodes = trie.NewMergedNodeSet() + ) + codeWriter := db.db.DiskDB().NewBatch() for addr := range db.stateObjectsDirty { if obj := db.stateObjects[addr]; !obj.deleted { // Write any contract code associated with the state object if obj.code != nil && obj.dirtyCode { - db.db.TrieDB().InsertBlob(common.BytesToHash(obj.CodeHash()), obj.code) + rawdb.WriteCode(codeWriter, common.BytesToHash(obj.CodeHash()), obj.code) obj.dirtyCode = false } // Write any storage changes in the state object to its storage trie - if err := obj.CommitTrie(db.db); err != nil { + set, err := obj.commitTrie(db.db) + if err != nil { return common.Hash{}, err } + // Merge the dirty nodes of storage trie into global set + if set != nil { + if err := nodes.Merge(set); err != nil { + return common.Hash{}, err + } + updates, deleted := set.Size() + storageTrieNodesUpdated += updates + storageTrieNodesDeleted += deleted + } } + // If the contract is destructed, the storage is still left in the + // database as dangling data. Theoretically it's should be wiped from + // database as well, but in hash-based-scheme it's extremely hard to + // determine that if the trie nodes are also referenced by other storage, + // and in path-based-scheme some technical challenges are still unsolved. + // Although it won't affect the correctness but please fix it TODO(rjl493456442). } if len(db.stateObjectsDirty) > 0 { db.stateObjectsDirty = make(map[common.Address]struct{}) } - // Write the account trie changes, measuing the amount of wasted time + if codeWriter.ValueSize() > 0 { + if err := codeWriter.Write(); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to commit dirty codes") + } + } + // Write the account trie changes, measuring the amount of wasted time + var start time.Time if metrics.EnabledExpensive { - defer func(start time.Time) { db.AccountCommits += time.Since(start) }(time.Now()) + start = time.Now() } - return db.trie.Commit(func(leaf []byte, parent common.Hash) error { - var account Account - if err := rlp.DecodeBytes(leaf, &account); err != nil { - return nil + root, set := db.trie.Commit(true) + // Merge the dirty nodes of account trie into global set + if set != nil { + if err := nodes.Merge(set); err != nil { + return common.Hash{}, err } - if account.Root != emptyRoot { - db.db.TrieDB().Reference(account.Root, parent) + accountTrieNodesUpdated, accountTrieNodesDeleted = set.Size() + } + if metrics.EnabledExpensive { + db.AccountCommits += time.Since(start) + + accountUpdatedMeter.Mark(int64(db.AccountUpdated)) + storageUpdatedMeter.Mark(int64(db.StorageUpdated)) + accountDeletedMeter.Mark(int64(db.AccountDeleted)) + storageDeletedMeter.Mark(int64(db.StorageDeleted)) + accountTrieUpdatedMeter.Mark(int64(accountTrieNodesUpdated)) + accountTrieDeletedMeter.Mark(int64(accountTrieNodesDeleted)) + storageTriesUpdatedMeter.Mark(int64(storageTrieNodesUpdated)) + storageTriesDeletedMeter.Mark(int64(storageTrieNodesDeleted)) + db.AccountUpdated, db.AccountDeleted = 0, 0 + db.StorageUpdated, db.StorageDeleted = 0, 0 + } + // If snapshotting is enabled, update the snapshot tree with this new version + if db.snap != nil { + start := time.Now() + // Only update if there's a state transition (skip empty Clique blocks) + if parent := db.snap.Root(); parent != root { + if err := db.snaps.Update(root, parent, db.convertAccountSet(db.stateObjectsDestruct), db.snapAccounts, db.snapStorage); err != nil { + utils.Logger().Warn().Err(err). + Interface("from", parent). + Interface("to", root). + Msg("Failed to update snapshot tree") + } + // Keep 128 diff layers in the memory, persistent layer is 129th. + // - head layer is paired with HEAD state + // - head-1 layer is paired with HEAD-1 state + // - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state + if err := db.snaps.Cap(root, 128); err != nil { + utils.Logger().Warn().Err(err). + Interface("root", root). + Uint16("layers", 128). + Msg("Failed to cap snapshot tree") + } } - code := common.BytesToHash(account.CodeHash) - if code != emptyCode { - db.db.TrieDB().Reference(code, parent) + if metrics.EnabledExpensive { + db.SnapshotCommits += time.Since(start) } - return nil - }) + db.snap, db.snapAccounts, db.snapStorage = nil, nil, nil + } + if len(db.stateObjectsDestruct) > 0 { + db.stateObjectsDestruct = make(map[common.Address]struct{}) + } + if root == (common.Hash{}) { + root = types.EmptyRootHash + } + origin := db.originalRoot + if origin == (common.Hash{}) { + origin = types.EmptyRootHash + } + if root != origin { + start := time.Now() + if err := db.db.TrieDB().Update(nodes); err != nil { + return common.Hash{}, err + } + db.originalRoot = root + if metrics.EnabledExpensive { + db.TrieDBCommits += time.Since(start) + } + } + return root, nil +} + +// AddAddressToAccessList adds the given address to the access list +func (db *DB) AddAddressToAccessList(addr common.Address) { + if db.accessList.AddAddress(addr) { + db.journal.append(accessListAddAccountChange{&addr}) + } +} + +// AddSlotToAccessList adds the given (address, slot)-tuple to the access list +func (db *DB) AddSlotToAccessList(addr common.Address, slot common.Hash) { + addrMod, slotMod := db.accessList.AddSlot(addr, slot) + if addrMod { + // In practice, this should not happen, since there is no way to enter the + // scope of 'address' without having the 'address' become already added + // to the access list (via call-variant, create, etc). + // Better safe than sorry, though + db.journal.append(accessListAddAccountChange{&addr}) + } + if slotMod { + db.journal.append(accessListAddSlotChange{ + address: &addr, + slot: &slot, + }) + } +} + +// AddressInAccessList returns true if the given address is in the access list. +func (db *DB) AddressInAccessList(addr common.Address) bool { + return db.accessList.ContainsAddress(addr) +} + +// SlotInAccessList returns true if the given (address, slot)-tuple is in the access list. +func (db *DB) SlotInAccessList(addr common.Address, slot common.Hash) (addressPresent bool, slotPresent bool) { + return db.accessList.Contains(addr, slot) +} + +// convertAccountSet converts a provided account set from address keyed to hash keyed. +func (db *DB) convertAccountSet(set map[common.Address]struct{}) map[common.Hash]struct{} { + ret := make(map[common.Hash]struct{}) + for addr := range set { + obj, exist := db.stateObjects[addr] + if !exist { + ret[crypto.Keccak256Hash(addr[:])] = struct{}{} + } else { + ret[obj.addrHash] = struct{}{} + } + } + return ret } var ( diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 9581495d54..db2cf5ebfd 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -29,19 +29,9 @@ import ( "testing" "testing/quick" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/types" - - "github.com/harmony-one/harmony/crypto/bls" - "github.com/harmony-one/harmony/crypto/hash" - "github.com/harmony-one/harmony/numeric" - stk "github.com/harmony-one/harmony/staking/types" - staketest "github.com/harmony-one/harmony/staking/types/test" - - "github.com/ethereum/go-ethereum/crypto" - "github.com/harmony-one/harmony/common/denominations" ) // Tests that updating a state trie does not leak any database writes prior to @@ -49,7 +39,7 @@ import ( func TestUpdateLeaks(t *testing.T) { // Create an empty state database db := rawdb.NewMemoryDatabase() - state, _ := New(common.Hash{}, NewDatabase(db)) + state, _ := New(common.Hash{}, NewDatabase(db), nil) // Update it with some accounts for i := byte(0); i < 255; i++ { @@ -70,7 +60,7 @@ func TestUpdateLeaks(t *testing.T) { } // Ensure that no data was leaked into the database - it := db.NewIterator() + it := db.NewIterator(nil, nil) for it.Next() { t.Errorf("State leaked into database: %x -> %x", it.Key(), it.Value()) } @@ -83,8 +73,8 @@ func TestIntermediateLeaks(t *testing.T) { // Create two state databases, one transitioning to the final state, the other final from the beginning transDb := rawdb.NewMemoryDatabase() finalDb := rawdb.NewMemoryDatabase() - transState, _ := New(common.Hash{}, NewDatabase(transDb)) - finalState, _ := New(common.Hash{}, NewDatabase(finalDb)) + transState, _ := New(common.Hash{}, NewDatabase(transDb), nil) + finalState, _ := New(common.Hash{}, NewDatabase(finalDb), nil) modify := func(state *DB, addr common.Address, i, tweak byte) { state.SetBalance(addr, big.NewInt(int64(11*i)+int64(tweak))) @@ -128,7 +118,7 @@ func TestIntermediateLeaks(t *testing.T) { t.Errorf("can not commit trie %v to persistent database", finalRoot.Hex()) } - it := finalDb.NewIterator() + it := finalDb.NewIterator(nil, nil) for it.Next() { key, fvalue := it.Key(), it.Value() tvalue, err := transDb.Get(key) @@ -136,12 +126,12 @@ func TestIntermediateLeaks(t *testing.T) { t.Errorf("entry missing from the transition database: %x -> %x", key, fvalue) } if !bytes.Equal(fvalue, tvalue) { - t.Errorf("the value associate key %x is mismatch,: %x in transition database ,%x in final database", key, tvalue, fvalue) + t.Errorf("value mismatch at key %x: %x in transition database, %x in final database", key, tvalue, fvalue) } } it.Release() - it = transDb.NewIterator() + it = transDb.NewIterator(nil, nil) for it.Next() { key, tvalue := it.Key(), it.Value() fvalue, err := finalDb.Get(key) @@ -149,29 +139,22 @@ func TestIntermediateLeaks(t *testing.T) { t.Errorf("extra entry in the transition database: %x -> %x", key, it.Value()) } if !bytes.Equal(fvalue, tvalue) { - t.Errorf("the value associate key %x is mismatch,: %x in transition database ,%x in final database", key, tvalue, fvalue) + t.Errorf("value mismatch at key %x: %x in transition database, %x in final database", key, tvalue, fvalue) } } } -// TestCopy tests that copying a statedb object indeed makes the original and +// TestCopy tests that copying a DB object indeed makes the original and // the copy independent of each other. This test is a regression test against // https://github.com/ethereum/go-ethereum/pull/15549. func TestCopy(t *testing.T) { // Create a random state test to copy and modify "independently" - orig, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + orig, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil) for i := byte(0); i < 255; i++ { obj := orig.GetOrNewStateObject(common.BytesToAddress([]byte{i})) obj.AddBalance(big.NewInt(int64(i))) orig.updateStateObject(obj) - - validatorWrapper := makeValidValidatorWrapper(common.BytesToAddress([]byte{i})) - validatorWrapper.Description.Name = "Original" - err := orig.UpdateValidatorWrapper(common.BytesToAddress([]byte{i}), &validatorWrapper) - if err != nil { - t.Errorf("Couldn't update ValidatorWrapper %d with error %s", i, err) - } } orig.Finalise(false) @@ -194,85 +177,6 @@ func TestCopy(t *testing.T) { orig.updateStateObject(origObj) copy.updateStateObject(copyObj) ccopy.updateStateObject(copyObj) - - origValWrap, err := orig.ValidatorWrapper(common.BytesToAddress([]byte{i}), false, true) - if err != nil { - t.Errorf("Couldn't get validatorWrapper with error: %s", err) - } - copyValWrap, err := copy.ValidatorWrapper(common.BytesToAddress([]byte{i}), false, true) - if err != nil { - t.Errorf("Couldn't get validatorWrapper with error: %s", err) - } - ccopyValWrap, err := ccopy.ValidatorWrapper(common.BytesToAddress([]byte{i}), false, true) - if err != nil { - t.Errorf("Couldn't get validatorWrapper with error: %s", err) - } - - origValWrap.LastEpochInCommittee.SetInt64(1) - copyValWrap.LastEpochInCommittee.SetInt64(2) - ccopyValWrap.LastEpochInCommittee.SetInt64(3) - - origValWrap.MinSelfDelegation.Mul(big.NewInt(1e18), big.NewInt(10000)) - copyValWrap.MinSelfDelegation.Mul(big.NewInt(1e18), big.NewInt(20000)) - ccopyValWrap.MinSelfDelegation.Mul(big.NewInt(1e18), big.NewInt(30000)) - - origValWrap.MaxTotalDelegation.Mul(big.NewInt(1e18), big.NewInt(10000)) - copyValWrap.MaxTotalDelegation.Mul(big.NewInt(1e18), big.NewInt(20000)) - ccopyValWrap.MaxTotalDelegation.Mul(big.NewInt(1e18), big.NewInt(30000)) - - origValWrap.CreationHeight.SetInt64(1) - copyValWrap.CreationHeight.SetInt64(2) - ccopyValWrap.CreationHeight.SetInt64(3) - - origValWrap.UpdateHeight.SetInt64(1) - copyValWrap.UpdateHeight.SetInt64(2) - ccopyValWrap.UpdateHeight.SetInt64(3) - - origValWrap.Description.Name = "UpdatedOriginal" + string(i) - copyValWrap.Description.Name = "UpdatedCopy" + string(i) - ccopyValWrap.Description.Name = "UpdatedCCopy" + string(i) - - origValWrap.Delegations[0].Amount.SetInt64(1) - copyValWrap.Delegations[0].Amount.SetInt64(2) - ccopyValWrap.Delegations[0].Amount.SetInt64(3) - - origValWrap.Delegations[0].Reward.SetInt64(1) - copyValWrap.Delegations[0].Reward.SetInt64(2) - ccopyValWrap.Delegations[0].Reward.SetInt64(3) - - origValWrap.Delegations[0].Undelegations[0].Amount.SetInt64(1) - copyValWrap.Delegations[0].Undelegations[0].Amount.SetInt64(2) - ccopyValWrap.Delegations[0].Undelegations[0].Amount.SetInt64(3) - - origValWrap.Delegations[0].Undelegations[0].Epoch.SetInt64(1) - copyValWrap.Delegations[0].Undelegations[0].Epoch.SetInt64(2) - ccopyValWrap.Delegations[0].Undelegations[0].Epoch.SetInt64(3) - - origValWrap.Counters.NumBlocksToSign.SetInt64(1) - copyValWrap.Counters.NumBlocksToSign.SetInt64(2) - ccopyValWrap.Counters.NumBlocksToSign.SetInt64(3) - - origValWrap.Counters.NumBlocksSigned.SetInt64(1) - copyValWrap.Counters.NumBlocksSigned.SetInt64(2) - ccopyValWrap.Counters.NumBlocksSigned.SetInt64(3) - - origValWrap.BlockReward.SetInt64(1) - copyValWrap.BlockReward.SetInt64(2) - ccopyValWrap.BlockReward.SetInt64(3) - - err = orig.UpdateValidatorWrapper(common.BytesToAddress([]byte{i}), origValWrap) - if err != nil { - t.Errorf("Couldn't update ValidatorWrapper %d with error %s", i, err) - } - err = copy.UpdateValidatorWrapper(common.BytesToAddress([]byte{i}), copyValWrap) - if err != nil { - t.Errorf("Couldn't update ValidatorWrapper %d with error %s", i, err) - } - err = ccopy.UpdateValidatorWrapper(common.BytesToAddress([]byte{i}), ccopyValWrap) - if err != nil { - t.Errorf("Couldn't update ValidatorWrapper %d with error %s", i, err) - } - } // Finalise the changes on all concurrently @@ -303,149 +207,6 @@ func TestCopy(t *testing.T) { if want := big.NewInt(5 * int64(i)); ccopyObj.Balance().Cmp(want) != 0 { t.Errorf("copy obj %d: balance mismatch: have %v, want %v", i, ccopyObj.Balance(), want) } - - origValWrap, err := orig.ValidatorWrapper(common.BytesToAddress([]byte{i}), true, false) - if err != nil { - t.Errorf("Couldn't get validatorWrapper %d with error: %s", i, err) - } - copyValWrap, err := copy.ValidatorWrapper(common.BytesToAddress([]byte{i}), true, false) - if err != nil { - t.Errorf("Couldn't get validatorWrapper %d with error: %s", i, err) - } - ccopyValWrap, err := ccopy.ValidatorWrapper(common.BytesToAddress([]byte{i}), true, false) - if err != nil { - t.Errorf("Couldn't get validatorWrapper %d with error: %s", i, err) - } - - if origValWrap.LastEpochInCommittee.Cmp(big.NewInt(1)) != 0 { - t.Errorf("LastEpochInCommittee %d: balance mismatch: have %v, want %v", i, origValWrap.LastEpochInCommittee, big.NewInt(1)) - } - if copyValWrap.LastEpochInCommittee.Cmp(big.NewInt(2)) != 0 { - t.Errorf("LastEpochInCommittee %d: balance mismatch: have %v, want %v", i, copyValWrap.LastEpochInCommittee, big.NewInt(2)) - } - if ccopyValWrap.LastEpochInCommittee.Cmp(big.NewInt(3)) != 0 { - t.Errorf("LastEpochInCommittee %d: balance mismatch: have %v, want %v", i, ccopyValWrap.LastEpochInCommittee, big.NewInt(3)) - } - - if want := new(big.Int).Mul(big.NewInt(1e18), big.NewInt(10000)); origValWrap.MinSelfDelegation.Cmp(want) != 0 { - t.Errorf("MinSelfDelegation %d: balance mismatch: have %v, want %v", i, origValWrap.MinSelfDelegation, want) - } - if want := new(big.Int).Mul(big.NewInt(1e18), big.NewInt(20000)); copyValWrap.MinSelfDelegation.Cmp(want) != 0 { - t.Errorf("MinSelfDelegation %d: balance mismatch: have %v, want %v", i, copyValWrap.MinSelfDelegation, want) - } - if want := new(big.Int).Mul(big.NewInt(1e18), big.NewInt(30000)); ccopyValWrap.MinSelfDelegation.Cmp(want) != 0 { - t.Errorf("MinSelfDelegation %d: balance mismatch: have %v, want %v", i, ccopyValWrap.MinSelfDelegation, want) - } - - if want := new(big.Int).Mul(big.NewInt(1e18), big.NewInt(10000)); origValWrap.MaxTotalDelegation.Cmp(want) != 0 { - t.Errorf("MaxTotalDelegation %d: balance mismatch: have %v, want %v", i, origValWrap.MaxTotalDelegation, want) - } - if want := new(big.Int).Mul(big.NewInt(1e18), big.NewInt(20000)); copyValWrap.MaxTotalDelegation.Cmp(want) != 0 { - t.Errorf("MaxTotalDelegation %d: balance mismatch: have %v, want %v", i, copyValWrap.MaxTotalDelegation, want) - } - if want := new(big.Int).Mul(big.NewInt(1e18), big.NewInt(30000)); ccopyValWrap.MaxTotalDelegation.Cmp(want) != 0 { - t.Errorf("MaxTotalDelegation %d: balance mismatch: have %v, want %v", i, ccopyValWrap.MaxTotalDelegation, want) - } - - if origValWrap.CreationHeight.Cmp(big.NewInt(1)) != 0 { - t.Errorf("CreationHeight %d: balance mismatch: have %v, want %v", i, origValWrap.CreationHeight, big.NewInt(1)) - } - if copyValWrap.CreationHeight.Cmp(big.NewInt(2)) != 0 { - t.Errorf("CreationHeight %d: balance mismatch: have %v, want %v", i, copyValWrap.CreationHeight, big.NewInt(2)) - } - if ccopyValWrap.CreationHeight.Cmp(big.NewInt(3)) != 0 { - t.Errorf("CreationHeight %d: balance mismatch: have %v, want %v", i, ccopyValWrap.CreationHeight, big.NewInt(3)) - } - - if origValWrap.UpdateHeight.Cmp(big.NewInt(1)) != 0 { - t.Errorf("UpdateHeight %d: balance mismatch: have %v, want %v", i, origValWrap.UpdateHeight, big.NewInt(1)) - } - if copyValWrap.UpdateHeight.Cmp(big.NewInt(2)) != 0 { - t.Errorf("UpdateHeight %d: balance mismatch: have %v, want %v", i, copyValWrap.UpdateHeight, big.NewInt(2)) - } - if ccopyValWrap.UpdateHeight.Cmp(big.NewInt(3)) != 0 { - t.Errorf("UpdateHeight %d: balance mismatch: have %v, want %v", i, ccopyValWrap.UpdateHeight, big.NewInt(3)) - } - - if want := "UpdatedOriginal" + string(i); origValWrap.Description.Name != want { - t.Errorf("originalValWrap %d: Incorrect Name: have %s, want %s", i, origValWrap.Description.Name, want) - } - if want := "UpdatedCopy" + string(i); copyValWrap.Description.Name != want { - t.Errorf("originalValWrap %d: Incorrect Name: have %s, want %s", i, copyValWrap.Description.Name, want) - } - if want := "UpdatedCCopy" + string(i); ccopyValWrap.Description.Name != want { - t.Errorf("originalValWrap %d: Incorrect Name: have %s, want %s", i, ccopyValWrap.Description.Name, want) - } - - if origValWrap.Delegations[0].Amount.Cmp(big.NewInt(1)) != 0 { - t.Errorf("Delegations[0].Amount %d: balance mismatch: have %v, want %v", i, origValWrap.Delegations[0].Amount, big.NewInt(1)) - } - if copyValWrap.Delegations[0].Amount.Cmp(big.NewInt(2)) != 0 { - t.Errorf("Delegations[0].Amount %d: balance mismatch: have %v, want %v", i, copyValWrap.Delegations[0].Amount, big.NewInt(2)) - } - if ccopyValWrap.Delegations[0].Amount.Cmp(big.NewInt(3)) != 0 { - t.Errorf("Delegations[0].Amount %d: balance mismatch: have %v, want %v", i, ccopyValWrap.Delegations[0].Amount, big.NewInt(3)) - } - - if origValWrap.Delegations[0].Reward.Cmp(big.NewInt(1)) != 0 { - t.Errorf("Delegations[0].Reward %d: balance mismatch: have %v, want %v", i, origValWrap.Delegations[0].Reward, big.NewInt(1)) - } - if copyValWrap.Delegations[0].Reward.Cmp(big.NewInt(2)) != 0 { - t.Errorf("Delegations[0].Reward %d: balance mismatch: have %v, want %v", i, copyValWrap.Delegations[0].Reward, big.NewInt(2)) - } - if ccopyValWrap.Delegations[0].Reward.Cmp(big.NewInt(3)) != 0 { - t.Errorf("Delegations[0].Reward %d: balance mismatch: have %v, want %v", i, ccopyValWrap.Delegations[0].Reward, big.NewInt(3)) - } - - if origValWrap.Delegations[0].Undelegations[0].Amount.Cmp(big.NewInt(1)) != 0 { - t.Errorf("Delegations[0].Undelegations[0].Amount %d: balance mismatch: have %v, want %v", i, origValWrap.Delegations[0].Undelegations[0].Amount, big.NewInt(1)) - } - if copyValWrap.Delegations[0].Undelegations[0].Amount.Cmp(big.NewInt(2)) != 0 { - t.Errorf("Delegations[0].Undelegations[0].Amount %d: balance mismatch: have %v, want %v", i, copyValWrap.Delegations[0].Undelegations[0].Amount, big.NewInt(2)) - } - if ccopyValWrap.Delegations[0].Undelegations[0].Amount.Cmp(big.NewInt(3)) != 0 { - t.Errorf("Delegations[0].Undelegations[0].Amount %d: balance mismatch: have %v, want %v", i, ccopyValWrap.Delegations[0].Undelegations[0].Amount, big.NewInt(3)) - } - - if origValWrap.Delegations[0].Undelegations[0].Epoch.Cmp(big.NewInt(1)) != 0 { - t.Errorf("CreationHeight %d: balance mismatch: have %v, want %v", i, origValWrap.Delegations[0].Undelegations[0].Epoch, big.NewInt(1)) - } - if copyValWrap.Delegations[0].Undelegations[0].Epoch.Cmp(big.NewInt(2)) != 0 { - t.Errorf("CreationHeight %d: balance mismatch: have %v, want %v", i, copyValWrap.Delegations[0].Undelegations[0].Epoch, big.NewInt(2)) - } - if ccopyValWrap.Delegations[0].Undelegations[0].Epoch.Cmp(big.NewInt(3)) != 0 { - t.Errorf("CreationHeight %d: balance mismatch: have %v, want %v", i, ccopyValWrap.Delegations[0].Undelegations[0].Epoch, big.NewInt(3)) - } - - if origValWrap.Counters.NumBlocksToSign.Cmp(big.NewInt(1)) != 0 { - t.Errorf("Counters.NumBlocksToSign %d: balance mismatch: have %v, want %v", i, origValWrap.Counters.NumBlocksToSign, big.NewInt(1)) - } - if copyValWrap.Counters.NumBlocksToSign.Cmp(big.NewInt(2)) != 0 { - t.Errorf("Counters.NumBlocksToSign %d: balance mismatch: have %v, want %v", i, copyValWrap.Counters.NumBlocksToSign, big.NewInt(2)) - } - if ccopyValWrap.Counters.NumBlocksToSign.Cmp(big.NewInt(3)) != 0 { - t.Errorf("Counters.NumBlocksToSign %d: balance mismatch: have %v, want %v", i, ccopyValWrap.Counters.NumBlocksToSign, big.NewInt(3)) - } - - if origValWrap.Counters.NumBlocksSigned.Cmp(big.NewInt(1)) != 0 { - t.Errorf("Counters.NumBlocksSigned %d: balance mismatch: have %v, want %v", i, origValWrap.Counters.NumBlocksSigned, big.NewInt(1)) - } - if copyValWrap.Counters.NumBlocksSigned.Cmp(big.NewInt(2)) != 0 { - t.Errorf("Counters.NumBlocksSigned %d: balance mismatch: have %v, want %v", i, copyValWrap.Counters.NumBlocksSigned, big.NewInt(2)) - } - if ccopyValWrap.Counters.NumBlocksSigned.Cmp(big.NewInt(3)) != 0 { - t.Errorf("Counters.NumBlocksSigned %d: balance mismatch: have %v, want %v", i, ccopyValWrap.Counters.NumBlocksSigned, big.NewInt(3)) - } - - if origValWrap.BlockReward.Cmp(big.NewInt(1)) != 0 { - t.Errorf("Block Reward %d: balance mismatch: have %v, want %v", i, origValWrap.BlockReward, big.NewInt(1)) - } - if copyValWrap.BlockReward.Cmp(big.NewInt(2)) != 0 { - t.Errorf("Block Reward %d: balance mismatch: have %v, want %v", i, copyValWrap.BlockReward, big.NewInt(2)) - } - if ccopyValWrap.BlockReward.Cmp(big.NewInt(3)) != 0 { - t.Errorf("Block Reward %d: balance mismatch: have %v, want %v", i, ccopyValWrap.BlockReward, big.NewInt(3)) - } } } @@ -460,7 +221,7 @@ func TestSnapshotRandom(t *testing.T) { } } -// A snapshotTest checks that reverting StateDB snapshots properly undoes all changes +// A snapshotTest checks that reverting DB snapshots properly undoes all changes // captured by the snapshot. Instances of this test with pseudorandom content are created // by Generate. // @@ -567,6 +328,30 @@ func newTestAction(addr common.Address, r *rand.Rand) testAction { }, args: make([]int64, 1), }, + { + name: "AddAddressToAccessList", + fn: func(a testAction, s *DB) { + s.AddAddressToAccessList(addr) + }, + }, + { + name: "AddSlotToAccessList", + fn: func(a testAction, s *DB) { + s.AddSlotToAccessList(addr, + common.Hash{byte(a.args[0])}) + }, + args: make([]int64, 1), + }, + { + name: "SetTransientState", + fn: func(a testAction, s *DB) { + var key, val common.Hash + binary.BigEndian.PutUint16(key[:], uint16(a.args[0])) + binary.BigEndian.PutUint16(val[:], uint16(a.args[1])) + s.SetTransientState(addr, key, val) + }, + args: make([]int64, 2), + }, } action := actions[r.Intn(len(actions))] var nameargs []string @@ -624,7 +409,7 @@ func (test *snapshotTest) String() string { func (test *snapshotTest) run() bool { // Run all actions and create snapshots. var ( - state, _ = New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + state, _ = New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil) snapshotRevs = make([]int, len(test.snapshots)) sindex = 0 ) @@ -638,7 +423,7 @@ func (test *snapshotTest) run() bool { // Revert all snapshots in reverse order. Each revert must yield a state // that is equivalent to fresh state with all actions up the snapshot applied. for sindex--; sindex >= 0; sindex-- { - checkstate, _ := New(common.Hash{}, state.Database()) + checkstate, _ := New(common.Hash{}, state.Database(), nil) for _, action := range test.actions[:test.snapshots[sindex]] { action.fn(action, checkstate) } @@ -688,9 +473,9 @@ func (test *snapshotTest) checkEqual(state, checkstate *DB) error { return fmt.Errorf("got GetRefund() == %d, want GetRefund() == %d", state.GetRefund(), checkstate.GetRefund()) } - if !reflect.DeepEqual(state.GetLogs(common.Hash{}), checkstate.GetLogs(common.Hash{})) { + if !reflect.DeepEqual(state.GetLogs(common.Hash{}, 0, common.Hash{}), checkstate.GetLogs(common.Hash{}, 0, common.Hash{})) { return fmt.Errorf("got GetLogs(common.Hash{}) == %v, want GetLogs(common.Hash{}) == %v", - state.GetLogs(common.Hash{}), checkstate.GetLogs(common.Hash{})) + state.GetLogs(common.Hash{}, 0, common.Hash{}), checkstate.GetLogs(common.Hash{}, 0, common.Hash{})) } return nil } @@ -699,7 +484,7 @@ func TestTouchDelete(t *testing.T) { s := newStateTest() s.state.GetOrNewStateObject(common.Address{}) root, _ := s.state.Commit(false) - s.state.Reset(root) + s.state, _ = New(root, s.state.db, s.state.snaps) snapshot := s.state.Snapshot() s.state.AddBalance(common.Address{}, new(big.Int)) @@ -716,7 +501,7 @@ func TestTouchDelete(t *testing.T) { // TestCopyOfCopy tests that modified objects are carried over to the copy, and the copy of the copy. // See https://github.com/ethereum/go-ethereum/pull/15225#issuecomment-380191512 func TestCopyOfCopy(t *testing.T) { - state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil) addr := common.HexToAddress("aaaa") state.SetBalance(addr, big.NewInt(42)) @@ -733,7 +518,7 @@ func TestCopyOfCopy(t *testing.T) { // // See https://github.com/ethereum/go-ethereum/issues/20106. func TestCopyCommitCopy(t *testing.T) { - state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil) // Create an account and check if the retrieved balance is correct addr := common.HexToAddress("0xaffeaffeaffeaffeaffeaffeaffeaffeaffeaffe") @@ -805,7 +590,7 @@ func TestCopyCommitCopy(t *testing.T) { // // See https://github.com/ethereum/go-ethereum/issues/20106. func TestCopyCopyCommitCopy(t *testing.T) { - state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil) // Create an account and check if the retrieved balance is correct addr := common.HexToAddress("0xaffeaffeaffeaffeaffeaffeaffeaffeaffeaffe") @@ -886,22 +671,22 @@ func TestCopyCopyCommitCopy(t *testing.T) { } // TestDeleteCreateRevert tests a weird state transition corner case that we hit -// while changing the internals of statedb. The workflow is that a contract is -// self destructed, then in a followup transaction (but same block) it's created +// while changing the internals of DB. The workflow is that a contract is +// self-destructed, then in a follow-up transaction (but same block) it's created // again and the transaction reverted. // -// The original statedb implementation flushed dirty objects to the tries after +// The original DB implementation flushed dirty objects to the tries after // each transaction, so this works ok. The rework accumulated writes in memory // first, but the journal wiped the entire state object on create-revert. func TestDeleteCreateRevert(t *testing.T) { // Create an initial state with a single contract - state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil) addr := common.BytesToAddress([]byte("so")) state.SetBalance(addr, big.NewInt(1)) root, _ := state.Commit(false) - state.Reset(root) + state, _ = New(root, state.db, state.snaps) // Simulate self-destructing in one transaction, then create-reverting in another state.Suicide(addr) @@ -913,315 +698,303 @@ func TestDeleteCreateRevert(t *testing.T) { // Commit the entire state and make sure we don't crash and have the correct state root, _ = state.Commit(true) - state.Reset(root) + state, _ = New(root, state.db, state.snaps) if state.getStateObject(addr) != nil { t.Fatalf("self-destructed contract came alive") } } -func makeValidValidatorWrapper(addr common.Address) stk.ValidatorWrapper { - cr := stk.CommissionRates{ - Rate: numeric.ZeroDec(), - MaxRate: numeric.ZeroDec(), - MaxChangeRate: numeric.ZeroDec(), - } - c := stk.Commission{CommissionRates: cr, UpdateHeight: big.NewInt(300)} - d := stk.Description{ - Name: "Wayne", - Identity: "wen", - Website: "harmony.one.wen", - Details: "best", - } - - v := stk.Validator{ - Address: addr, - SlotPubKeys: []bls.SerializedPublicKey{makeBLSPubSigPair().pub}, - LastEpochInCommittee: big.NewInt(20), - MinSelfDelegation: new(big.Int).Mul(big.NewInt(10000), big.NewInt(1e18)), - MaxTotalDelegation: new(big.Int).Mul(big.NewInt(12000), big.NewInt(1e18)), - Commission: c, - Description: d, - CreationHeight: big.NewInt(12306), - } - ds := stk.Delegations{ - stk.Delegation{ - DelegatorAddress: v.Address, - Amount: big.NewInt(0), - Reward: big.NewInt(0), - Undelegations: stk.Undelegations{ - stk.Undelegation{ - Amount: big.NewInt(0), - Epoch: big.NewInt(0), - }, - }, - }, +// TestMissingTrieNodes tests that if the DB fails to load parts of the trie, +// the Commit operation fails with an error +// If we are missing trie nodes, we should not continue writing to the trie +func TestMissingTrieNodes(t *testing.T) { + // Create an initial state with a few accounts + memDb := rawdb.NewMemoryDatabase() + db := NewDatabase(memDb) + var root common.Hash + state, _ := New(common.Hash{}, db, nil) + addr := common.BytesToAddress([]byte("so")) + { + state.SetBalance(addr, big.NewInt(1)) + state.SetCode(addr, []byte{1, 2, 3}) + a2 := common.BytesToAddress([]byte("another")) + state.SetBalance(a2, big.NewInt(100)) + state.SetCode(a2, []byte{1, 2, 4}) + root, _ = state.Commit(false) + t.Logf("root: %x", root) + // force-flush + state.Database().TrieDB().Cap(0) + } + // Create a new state on the old root + state, _ = New(root, db, nil) + // Now we clear out the memdb + it := memDb.NewIterator(nil, nil) + for it.Next() { + k := it.Key() + // Leave the root intact + if !bytes.Equal(k, root[:]) { + t.Logf("key: %x", k) + memDb.Delete(k) + } } - br := big.NewInt(1) - - w := stk.ValidatorWrapper{ - Validator: v, - Delegations: ds, - BlockReward: br, + balance := state.GetBalance(addr) + // The removed elem should lead to it returning zero balance + if exp, got := uint64(0), balance.Uint64(); got != exp { + t.Errorf("expected %d, got %d", exp, got) + } + // Modify the state + state.SetBalance(addr, big.NewInt(2)) + root, err := state.Commit(false) + if err == nil { + t.Fatalf("expected error, got root :%x", root) } - w.Counters.NumBlocksSigned = big.NewInt(0) - w.Counters.NumBlocksToSign = big.NewInt(0) - return w } -type blsPubSigPair struct { - pub bls.SerializedPublicKey - sig bls.SerializedSignature -} +func TestStateDBAccessList(t *testing.T) { + // Some helpers + addr := func(a string) common.Address { + return common.HexToAddress(a) + } + slot := func(a string) common.Hash { + return common.HexToHash(a) + } + + memDb := rawdb.NewMemoryDatabase() + db := NewDatabase(memDb) + state, _ := New(common.Hash{}, db, nil) + state.accessList = newAccessList() + + verifyAddrs := func(astrings ...string) { + t.Helper() + // convert to common.Address form + var addresses []common.Address + var addressMap = make(map[common.Address]struct{}) + for _, astring := range astrings { + address := addr(astring) + addresses = append(addresses, address) + addressMap[address] = struct{}{} + } + // Check that the given addresses are in the access list + for _, address := range addresses { + if !state.AddressInAccessList(address) { + t.Fatalf("expected %x to be in access list", address) + } + } + // Check that only the expected addresses are present in the access list + for address := range state.accessList.addresses { + if _, exist := addressMap[address]; !exist { + t.Fatalf("extra address %x in access list", address) + } + } + } + verifySlots := func(addrString string, slotStrings ...string) { + if !state.AddressInAccessList(addr(addrString)) { + t.Fatalf("scope missing address/slots %v", addrString) + } + var address = addr(addrString) + // convert to common.Hash form + var slots []common.Hash + var slotMap = make(map[common.Hash]struct{}) + for _, slotString := range slotStrings { + s := slot(slotString) + slots = append(slots, s) + slotMap[s] = struct{}{} + } + // Check that the expected items are in the access list + for i, s := range slots { + if _, slotPresent := state.SlotInAccessList(address, s); !slotPresent { + t.Fatalf("input %d: scope missing slot %v (address %v)", i, s, addrString) + } + } + // Check that no extra elements are in the access list + index := state.accessList.addresses[address] + if index >= 0 { + stateSlots := state.accessList.slots[index] + for s := range stateSlots { + if _, slotPresent := slotMap[s]; !slotPresent { + t.Fatalf("scope has extra slot %v (address %v)", s, addrString) + } + } + } + } -func makeBLSPubSigPair() blsPubSigPair { - blsPriv := bls.RandPrivateKey() - blsPub := blsPriv.GetPublicKey() - msgHash := hash.Keccak256([]byte(stk.BLSVerificationStr)) - sig := blsPriv.SignHash(msgHash) + state.AddAddressToAccessList(addr("aa")) // 1 + state.AddSlotToAccessList(addr("bb"), slot("01")) // 2,3 + state.AddSlotToAccessList(addr("bb"), slot("02")) // 4 + verifyAddrs("aa", "bb") + verifySlots("bb", "01", "02") - var shardPub bls.SerializedPublicKey - copy(shardPub[:], blsPub.Serialize()) + // Make a copy + stateCopy1 := state.Copy() + if exp, got := 4, state.journal.length(); exp != got { + t.Fatalf("journal length mismatch: have %d, want %d", got, exp) + } - var shardSig bls.SerializedSignature - copy(shardSig[:], sig.Serialize()) + // same again, should cause no journal entries + state.AddSlotToAccessList(addr("bb"), slot("01")) + state.AddSlotToAccessList(addr("bb"), slot("02")) + state.AddAddressToAccessList(addr("aa")) + if exp, got := 4, state.journal.length(); exp != got { + t.Fatalf("journal length mismatch: have %d, want %d", got, exp) + } + // some new ones + state.AddSlotToAccessList(addr("bb"), slot("03")) // 5 + state.AddSlotToAccessList(addr("aa"), slot("01")) // 6 + state.AddSlotToAccessList(addr("cc"), slot("01")) // 7,8 + state.AddAddressToAccessList(addr("cc")) + if exp, got := 8, state.journal.length(); exp != got { + t.Fatalf("journal length mismatch: have %d, want %d", got, exp) + } - return blsPubSigPair{shardPub, shardSig} -} + verifyAddrs("aa", "bb", "cc") + verifySlots("aa", "01") + verifySlots("bb", "01", "02", "03") + verifySlots("cc", "01") -func updateAndCheckValidator(t *testing.T, state *DB, wrapper stk.ValidatorWrapper) { - // do not modify / link the original into the state object - copied := staketest.CopyValidatorWrapper(wrapper) - if err := state.UpdateValidatorWrapperWithRevert(copied.Address, &copied); err != nil { - t.Fatalf("Could not update wrapper with revert %v\n", err) + // now start rolling back changes + state.journal.revert(state, 7) + if _, ok := state.SlotInAccessList(addr("cc"), slot("01")); ok { + t.Fatalf("slot present, expected missing") } + verifyAddrs("aa", "bb", "cc") + verifySlots("aa", "01") + verifySlots("bb", "01", "02", "03") - // load a copy here to be safe - loadedWrapper, err := state.ValidatorWrapper(copied.Address, false, true) - if err != nil { - t.Fatalf("Could not load wrapper %v\n", err) + state.journal.revert(state, 6) + if state.AddressInAccessList(addr("cc")) { + t.Fatalf("addr present, expected missing") } + verifyAddrs("aa", "bb") + verifySlots("aa", "01") + verifySlots("bb", "01", "02", "03") - if err := staketest.CheckValidatorWrapperEqual(wrapper, *loadedWrapper); err != nil { - t.Fatalf("Wrappers are unequal %v\n", err) + state.journal.revert(state, 5) + if _, ok := state.SlotInAccessList(addr("aa"), slot("01")); ok { + t.Fatalf("slot present, expected missing") } -} + verifyAddrs("aa", "bb") + verifySlots("bb", "01", "02", "03") -func verifyValidatorWrapperRevert( - t *testing.T, - state *DB, - snapshot int, - wrapperAddress common.Address, // if expectedWrapper is nil, this is needed - allowErrAddressNotPresent bool, - expectedWrapper *stk.ValidatorWrapper, - stateToCompare *DB, - modifiedAddresses []common.Address, -) { - state.RevertToSnapshot(snapshot) - loadedWrapper, err := state.ValidatorWrapper(wrapperAddress, true, false) - if err != nil && !(err == ErrAddressNotPresent && allowErrAddressNotPresent) { - t.Fatalf("Could not load wrapper %v\n", err) - } - if expectedWrapper != nil { - if err := staketest.CheckValidatorWrapperEqual(*expectedWrapper, *loadedWrapper); err != nil { - fmt.Printf("ExpectWrapper: %v\n", expectedWrapper) - fmt.Printf("LoadedWrapper: %v\n", loadedWrapper) - fmt.Printf("ExpectCounters: %v\n", expectedWrapper.Counters) - fmt.Printf("LoadedCounters: %v\n", loadedWrapper.Counters) - t.Fatalf("Loaded wrapper not equal to expected wrapper after revert %v\n", err) - } - } else if loadedWrapper != nil { - t.Fatalf("Expected wrapper is nil but got loaded wrapper %v\n", loadedWrapper) + state.journal.revert(state, 4) + if _, ok := state.SlotInAccessList(addr("bb"), slot("03")); ok { + t.Fatalf("slot present, expected missing") } + verifyAddrs("aa", "bb") + verifySlots("bb", "01", "02") - st := &snapshotTest{addrs: modifiedAddresses} - if err := st.checkEqual(state, stateToCompare); err != nil { - t.Fatalf("State not as expected after revert %v\n", err) + state.journal.revert(state, 3) + if _, ok := state.SlotInAccessList(addr("bb"), slot("02")); ok { + t.Fatalf("slot present, expected missing") } -} + verifyAddrs("aa", "bb") + verifySlots("bb", "01") -func TestValidatorCreationRevert(t *testing.T) { - state, err := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) - emptyState := state.Copy() - if err != nil { - t.Fatalf("Could not instantiate state %v\n", err) + state.journal.revert(state, 2) + if _, ok := state.SlotInAccessList(addr("bb"), slot("01")); ok { + t.Fatalf("slot present, expected missing") } - snapshot := state.Snapshot() - key, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("Could not generate key %v\n", err) - } - wrapper := makeValidValidatorWrapper(crypto.PubkeyToAddress(key.PublicKey)) - // step 1 is adding the validator, and checking that is it successfully added - updateAndCheckValidator(t, state, wrapper) - // step 2 is the revert check, the meat of the test - verifyValidatorWrapperRevert(t, - state, - snapshot, - wrapper.Address, - true, - nil, - emptyState, - []common.Address{wrapper.Address}, - ) -} + verifyAddrs("aa", "bb") -func TestValidatorAddDelegationRevert(t *testing.T) { - state, err := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) - if err != nil { - t.Fatalf("Could not instantiate state %v\n", err) - } - key, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("Could not generate key %v\n", err) + state.journal.revert(state, 1) + if state.AddressInAccessList(addr("bb")) { + t.Fatalf("addr present, expected missing") } - delegatorKey, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("Could not generate key %v\n", err) - } - wrapper := makeValidValidatorWrapper(crypto.PubkeyToAddress(key.PublicKey)) - // always, step 1 is adding the validator, and checking that is it successfully added - updateAndCheckValidator(t, state, wrapper) - wrapperWithoutDelegation := staketest.CopyValidatorWrapper(wrapper) // for comparison later - stateWithoutDelegation := state.Copy() - // we will revert to the state without the delegation - snapshot := state.Snapshot() - // which is added here - wrapper.Delegations = append(wrapper.Delegations, stk.NewDelegation( - crypto.PubkeyToAddress(delegatorKey.PublicKey), - new(big.Int).Mul(big.NewInt(denominations.One), big.NewInt(100))), - ) - // again, add and make sure added == sent - updateAndCheckValidator(t, state, wrapper) - // now the meat of the test - verifyValidatorWrapperRevert(t, - state, - snapshot, - wrapper.Address, - false, - &wrapperWithoutDelegation, - stateWithoutDelegation, - []common.Address{wrapper.Address, wrapper.Delegations[1].DelegatorAddress}, - ) -} - -type expectedRevertItem struct { - snapshot int - expectedWrapperAfterRevert *stk.ValidatorWrapper - expectedStateAfterRevert *DB - modifiedAddresses []common.Address -} + verifyAddrs("aa") -func makeExpectedRevertItem(state *DB, - wrapper *stk.ValidatorWrapper, - modifiedAddresses []common.Address, -) expectedRevertItem { - x := expectedRevertItem{ - snapshot: state.Snapshot(), - expectedStateAfterRevert: state.Copy(), - modifiedAddresses: modifiedAddresses, + state.journal.revert(state, 0) + if state.AddressInAccessList(addr("aa")) { + t.Fatalf("addr present, expected missing") + } + if got, exp := len(state.accessList.addresses), 0; got != exp { + t.Fatalf("expected empty, got %d", got) + } + if got, exp := len(state.accessList.slots), 0; got != exp { + t.Fatalf("expected empty, got %d", got) } - if wrapper != nil { - copied := staketest.CopyValidatorWrapper(*wrapper) - x.expectedWrapperAfterRevert = &copied + // Check the copy + // Make a copy + state = stateCopy1 + verifyAddrs("aa", "bb") + verifySlots("bb", "01", "02") + if got, exp := len(state.accessList.addresses), 2; got != exp { + t.Fatalf("expected empty, got %d", got) + } + if got, exp := len(state.accessList.slots), 1; got != exp { + t.Fatalf("expected empty, got %d", got) } - return x } -func TestValidatorMultipleReverts(t *testing.T) { - var expectedRevertItems []expectedRevertItem - - state, err := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) - if err != nil { - t.Fatalf("Could not instantiate state %v\n", err) +// Tests that account and storage tries are flushed in the correct order and that +// no data loss occurs. +func TestFlushOrderDataLoss(t *testing.T) { + // Create a state trie with many accounts and slots + var ( + memdb = rawdb.NewMemoryDatabase() + statedb = NewDatabase(memdb) + state, _ = New(common.Hash{}, statedb, nil) + ) + for a := byte(0); a < 10; a++ { + state.CreateAccount(common.Address{a}) + for s := byte(0); s < 10; s++ { + state.SetState(common.Address{a}, common.Hash{a, s}, common.Hash{a, s}) + } } - key, err := crypto.GenerateKey() + root, err := state.Commit(false) if err != nil { - t.Fatalf("Could not generate key %v\n", err) + t.Fatalf("failed to commit state trie: %v", err) } - delegatorKey, err := crypto.GenerateKey() + statedb.TrieDB().Reference(root, common.Hash{}) + if err := statedb.TrieDB().Cap(1024); err != nil { + t.Fatalf("failed to cap trie dirty cache: %v", err) + } + if err := statedb.TrieDB().Commit(root, false); err != nil { + t.Fatalf("failed to commit state trie: %v", err) + } + // Reopen the state trie from flushed disk and verify it + state, err = New(root, NewDatabase(memdb), nil) if err != nil { - t.Fatalf("Could not generate key %v\n", err) - } - validatorAddress := crypto.PubkeyToAddress(key.PublicKey) - delegatorAddress := crypto.PubkeyToAddress(delegatorKey.PublicKey) - modifiedAddresses := []common.Address{validatorAddress, delegatorAddress} - // first we add a validator - expectedRevertItems = append(expectedRevertItems, - makeExpectedRevertItem(state, nil, modifiedAddresses)) - wrapper := makeValidValidatorWrapper(crypto.PubkeyToAddress(key.PublicKey)) - updateAndCheckValidator(t, state, wrapper) - // then we add a delegation - expectedRevertItems = append(expectedRevertItems, - makeExpectedRevertItem(state, &wrapper, modifiedAddresses)) - wrapper.Delegations = append(wrapper.Delegations, stk.NewDelegation( - crypto.PubkeyToAddress(delegatorKey.PublicKey), - new(big.Int).Mul(big.NewInt(denominations.One), big.NewInt(100))), - ) - updateAndCheckValidator(t, state, wrapper) - // then we have it sign blocks - wrapper.Counters.NumBlocksToSign.Add( - wrapper.Counters.NumBlocksToSign, common.Big1, - ) - wrapper.Counters.NumBlocksSigned.Add( - wrapper.Counters.NumBlocksSigned, common.Big1, - ) - updateAndCheckValidator(t, state, wrapper) - // then modify the name and the block reward - expectedRevertItems = append(expectedRevertItems, - makeExpectedRevertItem(state, &wrapper, modifiedAddresses)) - wrapper.BlockReward.SetInt64(1) - wrapper.Validator.Description.Name = "Name" - for i := len(expectedRevertItems) - 1; i >= 0; i-- { - item := expectedRevertItems[i] - verifyValidatorWrapperRevert(t, - state, - item.snapshot, - wrapper.Address, - i == 0, - item.expectedWrapperAfterRevert, - item.expectedStateAfterRevert, - []common.Address{wrapper.Address}, - ) + t.Fatalf("failed to reopen state trie: %v", err) + } + for a := byte(0); a < 10; a++ { + for s := byte(0); s < 10; s++ { + if have := state.GetState(common.Address{a}, common.Hash{a, s}); have != (common.Hash{a, s}) { + t.Errorf("account %d: slot %d: state mismatch: have %x, want %x", a, s, have, common.Hash{a, s}) + } + } } } -func TestValidatorWrapperPanic(t *testing.T) { - defer func() { recover() }() +func TestStateDBTransientStorage(t *testing.T) { + memDb := rawdb.NewMemoryDatabase() + db := NewDatabase(memDb) + state, _ := New(common.Hash{}, db, nil) - state, err := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) - if err != nil { - t.Fatalf("Could not instantiate state %v\n", err) + key := common.Hash{0x01} + value := common.Hash{0x02} + addr := common.Address{} + + state.SetTransientState(addr, key, value) + if exp, got := 1, state.journal.length(); exp != got { + t.Fatalf("journal length mismatch: have %d, want %d", got, exp) } - key, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("Could not generate key %v\n", err) + // the retrieved value should equal what was set + if got := state.GetTransientState(addr, key); got != value { + t.Fatalf("transient storage mismatch: have %x, want %x", got, value) } - validatorAddress := crypto.PubkeyToAddress(key.PublicKey) - // will panic because we are asking for Original with copy of delegations - _, _ = state.ValidatorWrapper(validatorAddress, true, true) - t.Fatalf("Did not panic") -} -func TestValidatorWrapperGetCode(t *testing.T) { - state, err := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) - if err != nil { - t.Fatalf("Could not instantiate state %v\n", err) + // revert the transient state being set and then check that the + // value is now the empty hash + state.journal.revert(state, 0) + if got, exp := state.GetTransientState(addr, key), (common.Hash{}); exp != got { + t.Fatalf("transient storage mismatch: have %x, want %x", got, exp) } - key, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("Could not generate key %v\n", err) - } - wrapper := makeValidValidatorWrapper(crypto.PubkeyToAddress(key.PublicKey)) - updateAndCheckValidator(t, state, wrapper) - // delete it from the cache so we can force it to use GetCode - delete(state.stateValidators, wrapper.Address) - loadedWrapper, err := state.ValidatorWrapper(wrapper.Address, false, false) - if err := staketest.CheckValidatorWrapperEqual(wrapper, *loadedWrapper); err != nil { - fmt.Printf("ExpectWrapper: %v\n", wrapper) - fmt.Printf("LoadedWrapper: %v\n", loadedWrapper) - fmt.Printf("ExpectCounters: %v\n", wrapper.Counters) - fmt.Printf("LoadedCounters: %v\n", loadedWrapper.Counters) - t.Fatalf("Loaded wrapper not equal to expected wrapper%v\n", err) + + // set transient state and then copy the statedb and ensure that + // the transient state is copied + state.SetTransientState(addr, key, value) + cpy := state.Copy() + if got := cpy.GetTransientState(addr, key); got != value { + t.Fatalf("transient storage mismatch: have %x, want %x", got, value) } } diff --git a/core/state/sync.go b/core/state/sync.go new file mode 100644 index 0000000000..75740e3e0d --- /dev/null +++ b/core/state/sync.go @@ -0,0 +1,56 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "bytes" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +// NewStateSync create a new state trie download scheduler. +func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(keys [][]byte, leaf []byte) error, scheme string) *trie.Sync { + // Register the storage slot callback if the external callback is specified. + var onSlot func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error + if onLeaf != nil { + onSlot = func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error { + return onLeaf(keys, leaf) + } + } + // Register the account callback to connect the state trie and the storage + // trie belongs to the contract. + var syncer *trie.Sync + onAccount := func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error { + if onLeaf != nil { + if err := onLeaf(keys, leaf); err != nil { + return err + } + } + var obj Account + if err := rlp.Decode(bytes.NewReader(leaf), &obj); err != nil { + return err + } + syncer.AddSubTrie(obj.Root, path, parent, parentPath, onSlot) + syncer.AddCodeEntry(common.BytesToHash(obj.CodeHash), path, parent, parentPath) + return nil + } + syncer = trie.NewSync(root, database, onAccount, scheme) + return syncer +} diff --git a/core/state/tikv_clean.go b/core/state/tikv_clean.go index a62d429014..1442c3a139 100644 --- a/core/state/tikv_clean.go +++ b/core/state/tikv_clean.go @@ -6,6 +6,7 @@ import ( "sync/atomic" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/harmony-one/harmony/internal/shardchain/tikv_manage" @@ -36,7 +37,7 @@ func (s *DB) DiffAndCleanCache(shardId uint32, to *DB) (int, error) { addrBytes := s.trie.GetKey(it.LeafKey()) addr := common.BytesToAddress(addrBytes) - var fromAccount, toAccount Account + var fromAccount, toAccount types.StateAccount if err := rlp.DecodeBytes(it.LeafBlob(), &fromAccount); err != nil { continue } @@ -53,8 +54,14 @@ func (s *DB) DiffAndCleanCache(shardId uint32, to *DB) (int, error) { } // create account difference iterator - fromAccountTrie := newObject(s, addr, fromAccount).getTrie(s.db) - toAccountTrie := newObject(to, addr, toAccount).getTrie(to.db) + fromAccountTrie, errFromAcc := newObject(s, addr, fromAccount).getTrie(s.db) + if errFromAcc != nil { + continue + } + toAccountTrie, errToAcc := newObject(to, addr, toAccount).getTrie(to.db) + if errToAcc != nil { + continue + } accountIt, _ := trie.NewDifferenceIterator(toAccountTrie.NodeIterator(nil), fromAccountTrie.NodeIterator(nil)) // parallel to delete data diff --git a/core/state/transient_storage.go b/core/state/transient_storage.go new file mode 100644 index 0000000000..66e563efa7 --- /dev/null +++ b/core/state/transient_storage.go @@ -0,0 +1,55 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "github.com/ethereum/go-ethereum/common" +) + +// transientStorage is a representation of EIP-1153 "Transient Storage". +type transientStorage map[common.Address]Storage + +// newTransientStorage creates a new instance of a transientStorage. +func newTransientStorage() transientStorage { + return make(transientStorage) +} + +// Set sets the transient-storage `value` for `key` at the given `addr`. +func (t transientStorage) Set(addr common.Address, key, value common.Hash) { + if _, ok := t[addr]; !ok { + t[addr] = make(Storage) + } + t[addr][key] = value +} + +// Get gets the transient storage for `key` at the given `addr`. +func (t transientStorage) Get(addr common.Address, key common.Hash) common.Hash { + val, ok := t[addr] + if !ok { + return common.Hash{} + } + return val[key] +} + +// Copy does a deep copy of the transientStorage +func (t transientStorage) Copy() transientStorage { + storage := make(transientStorage) + for key, value := range t { + storage[key] = value.Copy() + } + return storage +} diff --git a/core/state/trie_prefetcher.go b/core/state/trie_prefetcher.go new file mode 100644 index 0000000000..0fac2dafbf --- /dev/null +++ b/core/state/trie_prefetcher.go @@ -0,0 +1,354 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/metrics" + "github.com/harmony-one/harmony/internal/utils" +) + +var ( + // triePrefetchMetricsPrefix is the prefix under which to publish the metrics. + triePrefetchMetricsPrefix = "trie/prefetch/" +) + +// triePrefetcher is an active prefetcher, which receives accounts or storage +// items and does trie-loading of them. The goal is to get as much useful content +// into the caches as possible. +// +// Note, the prefetcher's API is not thread safe. +type triePrefetcher struct { + db Database // Database to fetch trie nodes through + root common.Hash // Root hash of the account trie for metrics + fetches map[string]Trie // Partially or fully fetcher tries + fetchers map[string]*subfetcher // Subfetchers for each trie + + deliveryMissMeter metrics.Meter + accountLoadMeter metrics.Meter + accountDupMeter metrics.Meter + accountSkipMeter metrics.Meter + accountWasteMeter metrics.Meter + storageLoadMeter metrics.Meter + storageDupMeter metrics.Meter + storageSkipMeter metrics.Meter + storageWasteMeter metrics.Meter +} + +func newTriePrefetcher(db Database, root common.Hash, namespace string) *triePrefetcher { + prefix := triePrefetchMetricsPrefix + namespace + p := &triePrefetcher{ + db: db, + root: root, + fetchers: make(map[string]*subfetcher), // Active prefetchers use the fetchers map + + deliveryMissMeter: metrics.GetOrRegisterMeter(prefix+"/deliverymiss", nil), + accountLoadMeter: metrics.GetOrRegisterMeter(prefix+"/account/load", nil), + accountDupMeter: metrics.GetOrRegisterMeter(prefix+"/account/dup", nil), + accountSkipMeter: metrics.GetOrRegisterMeter(prefix+"/account/skip", nil), + accountWasteMeter: metrics.GetOrRegisterMeter(prefix+"/account/waste", nil), + storageLoadMeter: metrics.GetOrRegisterMeter(prefix+"/storage/load", nil), + storageDupMeter: metrics.GetOrRegisterMeter(prefix+"/storage/dup", nil), + storageSkipMeter: metrics.GetOrRegisterMeter(prefix+"/storage/skip", nil), + storageWasteMeter: metrics.GetOrRegisterMeter(prefix+"/storage/waste", nil), + } + return p +} + +// close iterates over all the subfetchers, aborts any that were left spinning +// and reports the stats to the metrics subsystem. +func (p *triePrefetcher) close() { + for _, fetcher := range p.fetchers { + fetcher.abort() // safe to do multiple times + + if metrics.Enabled { + if fetcher.root == p.root { + p.accountLoadMeter.Mark(int64(len(fetcher.seen))) + p.accountDupMeter.Mark(int64(fetcher.dups)) + p.accountSkipMeter.Mark(int64(len(fetcher.tasks))) + + for _, key := range fetcher.used { + delete(fetcher.seen, string(key)) + } + p.accountWasteMeter.Mark(int64(len(fetcher.seen))) + } else { + p.storageLoadMeter.Mark(int64(len(fetcher.seen))) + p.storageDupMeter.Mark(int64(fetcher.dups)) + p.storageSkipMeter.Mark(int64(len(fetcher.tasks))) + + for _, key := range fetcher.used { + delete(fetcher.seen, string(key)) + } + p.storageWasteMeter.Mark(int64(len(fetcher.seen))) + } + } + } + // Clear out all fetchers (will crash on a second call, deliberate) + p.fetchers = nil +} + +// copy creates a deep-but-inactive copy of the trie prefetcher. Any trie data +// already loaded will be copied over, but no goroutines will be started. This +// is mostly used in the miner which creates a copy of it's actively mutated +// state to be sealed while it may further mutate the state. +func (p *triePrefetcher) copy() *triePrefetcher { + copy := &triePrefetcher{ + db: p.db, + root: p.root, + fetches: make(map[string]Trie), // Active prefetchers use the fetches map + + deliveryMissMeter: p.deliveryMissMeter, + accountLoadMeter: p.accountLoadMeter, + accountDupMeter: p.accountDupMeter, + accountSkipMeter: p.accountSkipMeter, + accountWasteMeter: p.accountWasteMeter, + storageLoadMeter: p.storageLoadMeter, + storageDupMeter: p.storageDupMeter, + storageSkipMeter: p.storageSkipMeter, + storageWasteMeter: p.storageWasteMeter, + } + // If the prefetcher is already a copy, duplicate the data + if p.fetches != nil { + for root, fetch := range p.fetches { + if fetch == nil { + continue + } + copy.fetches[root] = p.db.CopyTrie(fetch) + } + return copy + } + // Otherwise we're copying an active fetcher, retrieve the current states + for id, fetcher := range p.fetchers { + copy.fetches[id] = fetcher.peek() + } + return copy +} + +// prefetch schedules a batch of trie items to prefetch. +func (p *triePrefetcher) prefetch(owner common.Hash, root common.Hash, keys [][]byte) { + // If the prefetcher is an inactive one, bail out + if p.fetches != nil { + return + } + // Active fetcher, schedule the retrievals + id := p.trieID(owner, root) + fetcher := p.fetchers[id] + if fetcher == nil { + fetcher = newSubfetcher(p.db, p.root, owner, root) + p.fetchers[id] = fetcher + } + fetcher.schedule(keys) +} + +// trie returns the trie matching the root hash, or nil if the prefetcher doesn't +// have it. +func (p *triePrefetcher) trie(owner common.Hash, root common.Hash) Trie { + // If the prefetcher is inactive, return from existing deep copies + id := p.trieID(owner, root) + if p.fetches != nil { + trie := p.fetches[id] + if trie == nil { + p.deliveryMissMeter.Mark(1) + return nil + } + return p.db.CopyTrie(trie) + } + // Otherwise the prefetcher is active, bail if no trie was prefetched for this root + fetcher := p.fetchers[id] + if fetcher == nil { + p.deliveryMissMeter.Mark(1) + return nil + } + // Interrupt the prefetcher if it's by any chance still running and return + // a copy of any pre-loaded trie. + fetcher.abort() // safe to do multiple times + + trie := fetcher.peek() + if trie == nil { + p.deliveryMissMeter.Mark(1) + return nil + } + return trie +} + +// used marks a batch of state items used to allow creating statistics as to +// how useful or wasteful the prefetcher is. +func (p *triePrefetcher) used(owner common.Hash, root common.Hash, used [][]byte) { + if fetcher := p.fetchers[p.trieID(owner, root)]; fetcher != nil { + fetcher.used = used + } +} + +// trieID returns an unique trie identifier consists the trie owner and root hash. +func (p *triePrefetcher) trieID(owner common.Hash, root common.Hash) string { + return string(append(owner.Bytes(), root.Bytes()...)) +} + +// subfetcher is a trie fetcher goroutine responsible for pulling entries for a +// single trie. It is spawned when a new root is encountered and lives until the +// main prefetcher is paused and either all requested items are processed or if +// the trie being worked on is retrieved from the prefetcher. +type subfetcher struct { + db Database // Database to load trie nodes through + state common.Hash // Root hash of the state to prefetch + owner common.Hash // Owner of the trie, usually account hash + root common.Hash // Root hash of the trie to prefetch + trie Trie // Trie being populated with nodes + + tasks [][]byte // Items queued up for retrieval + lock sync.Mutex // Lock protecting the task queue + + wake chan struct{} // Wake channel if a new task is scheduled + stop chan struct{} // Channel to interrupt processing + term chan struct{} // Channel to signal interruption + copy chan chan Trie // Channel to request a copy of the current trie + + seen map[string]struct{} // Tracks the entries already loaded + dups int // Number of duplicate preload tasks + used [][]byte // Tracks the entries used in the end +} + +// newSubfetcher creates a goroutine to prefetch state items belonging to a +// particular root hash. +func newSubfetcher(db Database, state common.Hash, owner common.Hash, root common.Hash) *subfetcher { + sf := &subfetcher{ + db: db, + state: state, + owner: owner, + root: root, + wake: make(chan struct{}, 1), + stop: make(chan struct{}), + term: make(chan struct{}), + copy: make(chan chan Trie), + seen: make(map[string]struct{}), + } + go sf.loop() + return sf +} + +// schedule adds a batch of trie keys to the queue to prefetch. +func (sf *subfetcher) schedule(keys [][]byte) { + // Append the tasks to the current queue + sf.lock.Lock() + sf.tasks = append(sf.tasks, keys...) + sf.lock.Unlock() + + // Notify the prefetcher, it's fine if it's already terminated + select { + case sf.wake <- struct{}{}: + default: + } +} + +// peek tries to retrieve a deep copy of the fetcher's trie in whatever form it +// is currently. +func (sf *subfetcher) peek() Trie { + ch := make(chan Trie) + select { + case sf.copy <- ch: + // Subfetcher still alive, return copy from it + return <-ch + + case <-sf.term: + // Subfetcher already terminated, return a copy directly + if sf.trie == nil { + return nil + } + return sf.db.CopyTrie(sf.trie) + } +} + +// abort interrupts the subfetcher immediately. It is safe to call abort multiple +// times but it is not thread safe. +func (sf *subfetcher) abort() { + select { + case <-sf.stop: + default: + close(sf.stop) + } + <-sf.term +} + +// loop waits for new tasks to be scheduled and keeps loading them until it runs +// out of tasks or its underlying trie is retrieved for committing. +func (sf *subfetcher) loop() { + // No matter how the loop stops, signal anyone waiting that it's terminated + defer close(sf.term) + + // Start by opening the trie and stop processing if it fails + if sf.owner == (common.Hash{}) { + trie, err := sf.db.OpenTrie(sf.root) + if err != nil { + utils.Logger().Warn().Err(err).Interface("root", sf.root).Msg("Trie prefetcher failed opening trie") + return + } + sf.trie = trie + } else { + trie, err := sf.db.OpenStorageTrie(sf.state, sf.owner, sf.root) + if err != nil { + utils.Logger().Warn().Err(err).Interface("root", sf.root).Msg("Trie prefetcher failed opening trie") + return + } + sf.trie = trie + } + // Trie opened successfully, keep prefetching items + for { + select { + case <-sf.wake: + // Subfetcher was woken up, retrieve any tasks to avoid spinning the lock + sf.lock.Lock() + tasks := sf.tasks + sf.tasks = nil + sf.lock.Unlock() + + // Prefetch any tasks until the loop is interrupted + for i, task := range tasks { + select { + case <-sf.stop: + // If termination is requested, add any leftover back and return + sf.lock.Lock() + sf.tasks = append(sf.tasks, tasks[i:]...) + sf.lock.Unlock() + return + + case ch := <-sf.copy: + // Somebody wants a copy of the current trie, grant them + ch <- sf.db.CopyTrie(sf.trie) + + default: + // No termination request yet, prefetch the next entry + if _, ok := sf.seen[string(task)]; ok { + sf.dups++ + } else { + sf.trie.TryGet(task) + sf.seen[string(task)] = struct{}{} + } + } + } + + case ch := <-sf.copy: + // Somebody wants a copy of the current trie, grant them + ch <- sf.db.CopyTrie(sf.trie) + + case <-sf.stop: + // Termination is requested, abort and leave remaining tasks + return + } + } +} diff --git a/core/state/trie_prefetcher_test.go b/core/state/trie_prefetcher_test.go new file mode 100644 index 0000000000..ffd67e02a8 --- /dev/null +++ b/core/state/trie_prefetcher_test.go @@ -0,0 +1,110 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/harmony/core/rawdb" +) + +func filledStateDB() *DB { + state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil) + + // Create an account and check if the retrieved balance is correct + addr := common.HexToAddress("0xaffeaffeaffeaffeaffeaffeaffeaffeaffeaffe") + skey := common.HexToHash("aaa") + sval := common.HexToHash("bbb") + + state.SetBalance(addr, big.NewInt(42)) // Change the account trie + state.SetCode(addr, []byte("hello")) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie + for i := 0; i < 100; i++ { + sk := common.BigToHash(big.NewInt(int64(i))) + state.SetState(addr, sk, sk) // Change the storage trie + } + return state +} + +func TestCopyAndClose(t *testing.T) { + db := filledStateDB() + prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") + skey := common.HexToHash("aaa") + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + time.Sleep(1 * time.Second) + a := prefetcher.trie(common.Hash{}, db.originalRoot) + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + b := prefetcher.trie(common.Hash{}, db.originalRoot) + cpy := prefetcher.copy() + cpy.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + cpy.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + c := cpy.trie(common.Hash{}, db.originalRoot) + prefetcher.close() + cpy2 := cpy.copy() + cpy2.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + d := cpy2.trie(common.Hash{}, db.originalRoot) + cpy.close() + cpy2.close() + if a.Hash() != b.Hash() || a.Hash() != c.Hash() || a.Hash() != d.Hash() { + t.Fatalf("Invalid trie, hashes should be equal: %v %v %v %v", a.Hash(), b.Hash(), c.Hash(), d.Hash()) + } +} + +func TestUseAfterClose(t *testing.T) { + db := filledStateDB() + prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") + skey := common.HexToHash("aaa") + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + a := prefetcher.trie(common.Hash{}, db.originalRoot) + prefetcher.close() + b := prefetcher.trie(common.Hash{}, db.originalRoot) + if a == nil { + t.Fatal("Prefetching before close should not return nil") + } + if b != nil { + t.Fatal("Trie after close should return nil") + } +} + +func TestCopyClose(t *testing.T) { + db := filledStateDB() + prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") + skey := common.HexToHash("aaa") + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + cpy := prefetcher.copy() + a := prefetcher.trie(common.Hash{}, db.originalRoot) + b := cpy.trie(common.Hash{}, db.originalRoot) + prefetcher.close() + c := prefetcher.trie(common.Hash{}, db.originalRoot) + d := cpy.trie(common.Hash{}, db.originalRoot) + if a == nil { + t.Fatal("Prefetching before close should not return nil") + } + if b == nil { + t.Fatal("Copy trie should return nil") + } + if c != nil { + t.Fatal("Trie after close should return nil") + } + if d == nil { + t.Fatal("Copy trie should not return nil") + } +} diff --git a/core/state_processor.go b/core/state_processor.go index 23c7e35389..fe7eeffd12 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -308,7 +308,7 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *commo // Set the receipt logs and create a bloom for filtering if config.IsReceiptLog(header.Epoch()) { - receipt.Logs = statedb.GetLogs(tx.Hash()) + receipt.Logs = statedb.GetLogs(tx.Hash(), header.Number().Uint64(), header.Hash()) } receipt.Bloom = types.CreateBloom(types.Receipts{receipt}) @@ -384,7 +384,7 @@ func ApplyStakingTransaction( receipt.GasUsed = gas if config.IsReceiptLog(header.Epoch()) { - receipt.Logs = statedb.GetLogs(tx.Hash()) + receipt.Logs = statedb.GetLogs(tx.Hash(), header.Number().Uint64(), header.Hash()) utils.Logger().Info().Interface("CollectReward", receipt.Logs) } diff --git a/core/tx_pool.go b/core/tx_pool.go index bd062b6f4c..d918490223 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -1436,7 +1436,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { if pending > pool.config.GlobalSlots { pendingBeforeCap := pending // Assemble a spam order to penalize large transactors first - spammers := prque.New(nil) + spammers := prque.New[int64, common.Address](nil) for addr, list := range pool.pending { // Only evict transactions from high rollers if !pool.locals.contains(addr) && uint64(list.Len()) > pool.config.AccountSlots { @@ -1448,12 +1448,12 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { for pending > pool.config.GlobalSlots && !spammers.Empty() { // Retrieve the next offender if not local address offender, _ := spammers.Pop() - offenders = append(offenders, offender.(common.Address)) + offenders = append(offenders, offender) // Equalize balances until all the same or below threshold if len(offenders) > 1 { // Calculate the equalization threshold for all current offenders - threshold := pool.pending[offender.(common.Address)].Len() + threshold := pool.pending[offender].Len() // Iteratively reduce all offenders until below limit or threshold reached for pending > pool.config.GlobalSlots && pool.pending[offenders[len(offenders)-2]].Len() > threshold { diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index fa3af1c972..5d11b50e9a 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -167,7 +167,7 @@ func createBlockChain() *BlockChainImpl { func setupTxPool(chain blockChain) (*TxPool, *ecdsa.PrivateKey) { if chain == nil { - statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) chain = &testBlockChain{statedb, 1e18, new(event.Feed)} } @@ -225,7 +225,7 @@ func (c *testChain) State() (*state.DB, error) { // a state change between those fetches. stdb := c.statedb if *c.trigger { - c.statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + c.statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) // simulate that the new head block included tx0 and tx1 c.statedb.SetNonce(c.address, 2) c.statedb.SetBalance(c.address, new(big.Int).SetUint64(denominations.One)) @@ -243,7 +243,7 @@ func TestStateChangeDuringTransactionPoolReset(t *testing.T) { var ( key, _ = crypto.GenerateKey() address = crypto.PubkeyToAddress(key.PublicKey) - statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) trigger = false ) @@ -579,7 +579,7 @@ func TestTransactionChainFork(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { - statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) statedb.AddBalance(addr, big.NewInt(9000000000000000000)) pool.chain = &testBlockChain{statedb, 1000000, new(event.Feed)} @@ -605,7 +605,7 @@ func TestTransactionDoubleNonce(t *testing.T) { key, _ := crypto.GenerateKey() addr := crypto.PubkeyToAddress(key.PublicKey) - statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) statedb.AddBalance(addr, big.NewInt(1000000000000000000)) pool, _ := setupTxPool(&testBlockChain{statedb, 1000000, new(event.Feed)}) defer pool.Stop() @@ -799,7 +799,7 @@ func TestTransactionPostponing(t *testing.T) { t.Parallel() // Create the pool to test the postponing with - statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain, dummyErrorSink) @@ -961,7 +961,7 @@ func testTransactionQueueGlobalLimiting(t *testing.T, nolocals bool) { t.Parallel() // Create the pool to test the limit enforcement with - statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1051,7 +1051,7 @@ func testTransactionQueueTimeLimiting(t *testing.T, nolocals bool) { evictionInterval = time.Second // Create the pool to test the non-expiration enforcement - statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1166,7 +1166,7 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1212,7 +1212,7 @@ func TestTransactionCapClearsFromAll(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1246,7 +1246,7 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1291,7 +1291,7 @@ func TestTransactionPoolRepricingKeepsLocals(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain, dummyErrorSink) @@ -1365,7 +1365,7 @@ func testTransactionJournaling(t *testing.T, nolocals bool) { os.Remove(journal) // Create the original pool to inject transaction into the journal - statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} config := testTxPoolConfig @@ -1463,7 +1463,7 @@ func TestTransactionStatusCheck(t *testing.T) { t.Parallel() // Create the pool to test the status retrievals with - statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain, dummyErrorSink) diff --git a/core/types/bloom9_test.go b/core/types/bloom9_test.go index 718a2078d2..4a4fcd5215 100644 --- a/core/types/bloom9_test.go +++ b/core/types/bloom9_test.go @@ -34,19 +34,19 @@ func TestBloom(t *testing.T) { "tes", "lo", } - var bloom ethtypes.Bloom for _, data := range positive { - bloom.Add(new(big.Int).SetBytes([]byte(data))) + b := new(big.Int).SetBytes([]byte(data)).Bytes() + bloom.Add(b) } for _, data := range positive { - if !bloom.TestBytes([]byte(data)) { + if !bloom.Test([]byte(data)) { t.Error("expected", data, "to test true") } } for _, data := range negative { - if bloom.TestBytes([]byte(data)) { + if bloom.Test([]byte(data)) { t.Error("did not expect", data, "to test true") } } diff --git a/core/vm/contracts_write_test.go b/core/vm/contracts_write_test.go index 649bbc6169..c7269e6fff 100644 --- a/core/vm/contracts_write_test.go +++ b/core/vm/contracts_write_test.go @@ -230,12 +230,12 @@ func testCrossShardXferPrecompile(test writeCapablePrecompileTest, t *testing.T) db.Close() } }() - if db, err = rawdb.NewLevelDBDatabase("/tmp/harmony_shard_0", 256, 1024, ""); err != nil { + if db, err = rawdb.NewLevelDBDatabase("/tmp/harmony_shard_0", 256, 1024, "", false); err != nil { db = nil t.Fatalf("Could not initialize db %s", err) } stateCache := state.NewDatabase(db) - state, err := state.New(common.Hash{}, stateCache) + state, err := state.New(common.Hash{}, stateCache, nil) if err != nil { t.Fatalf("Error while initializing state %s", err) } diff --git a/core/vm/gas_table_test.go b/core/vm/gas_table_test.go index 48aa846a0c..4619ccd029 100644 --- a/core/vm/gas_table_test.go +++ b/core/vm/gas_table_test.go @@ -83,7 +83,7 @@ func TestEIP2200(t *testing.T) { for i, tt := range eip2200Tests { address := common.BytesToAddress([]byte("contract")) - statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) statedb.CreateAccount(address) statedb.SetCode(address, hexutil.MustDecode(tt.input)) statedb.SetState(address, common.Hash{}, common.BytesToHash([]byte{tt.original})) diff --git a/core/vm/logger_test.go b/core/vm/logger_test.go index 1fb44a94fd..01db3293b7 100644 --- a/core/vm/logger_test.go +++ b/core/vm/logger_test.go @@ -21,7 +21,6 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/internal/params" ) @@ -43,7 +42,7 @@ func (d *dummyContractRef) SetNonce(uint64) {} func (d *dummyContractRef) Balance() *big.Int { return new(big.Int) } type dummyStatedb struct { - state.DB + StateDB } func (*dummyStatedb) GetRefund() uint64 { return 1337 } diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index 8ebba4030c..69275fc58f 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -103,7 +103,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.DB, error) { setDefaults(cfg) if cfg.State == nil { - cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) } var ( address = common.BytesToAddress([]byte("contract")) @@ -133,7 +133,7 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, uint64, error) { setDefaults(cfg) if cfg.State == nil { - cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) } var ( vmenv = NewEnv(cfg) diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 39d655fddc..09f2b6fc49 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -96,7 +96,7 @@ func TestExecute(t *testing.T) { } func TestCall(t *testing.T) { - state, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + state, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) address := common.HexToAddress("0x0a") state.SetCode(address, []byte{ byte(vm.PUSH1), 10, @@ -152,7 +152,7 @@ func BenchmarkCall(b *testing.B) { } func benchmarkEVMCreate(bench *testing.B, code string) { var ( - statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) sender = common.BytesToAddress([]byte("sender")) receiver = common.BytesToAddress([]byte("receiver")) ) diff --git a/go.mod b/go.mod index 18e16a0c05..2a2cabbbfc 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( github.com/RoaringBitmap/roaring v1.2.3 - github.com/VictoriaMetrics/fastcache v1.5.7 + github.com/VictoriaMetrics/fastcache v1.12.1 github.com/Workiva/go-datastructures v1.0.50 github.com/allegro/bigcache v1.2.1 github.com/aws/aws-sdk-go v1.34.0 @@ -13,9 +13,8 @@ require ( github.com/cespare/cp v1.1.1 github.com/coinbase/rosetta-sdk-go v0.7.0 github.com/davecgh/go-spew v1.1.1 - github.com/deckarep/golang-set v1.7.1 - github.com/ethereum/go-ethereum v1.9.25 - github.com/fjl/memsize v0.0.0-20180929194037-2a09253e352a // indirect + github.com/deckarep/golang-set v1.8.0 + github.com/ethereum/go-ethereum v1.11.2 github.com/go-redis/redis/v8 v8.11.5 github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.2 @@ -44,7 +43,7 @@ require ( github.com/rjeczalik/notify v0.9.2 github.com/rs/cors v1.7.0 github.com/rs/zerolog v1.18.0 - github.com/spf13/cobra v0.0.5 + github.com/spf13/cobra v1.5.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.14.0 github.com/stretchr/testify v1.8.1 @@ -53,7 +52,7 @@ require ( github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee go.uber.org/ratelimit v0.1.0 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.4.0 + golang.org/x/crypto v0.6.0 golang.org/x/net v0.7.0 // indirect golang.org/x/sync v0.1.0 golang.org/x/sys v0.5.0 // indirect @@ -72,24 +71,31 @@ require ( github.com/ledgerwatch/erigon-lib v0.0.0-20221218022306-0f8fdd40c2db github.com/ledgerwatch/log/v3 v3.6.0 github.com/libp2p/go-libp2p-core v0.20.1 + github.com/olekukonko/tablewriter v0.0.5 ) require ( github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect - github.com/BurntSushi/toml v0.3.1 // indirect + github.com/BurntSushi/toml v1.2.0 // indirect + github.com/DataDog/zstd v1.5.2 // indirect github.com/OpenPeeDeeP/depguard v1.0.1 // indirect github.com/VictoriaMetrics/metrics v1.23.0 // indirect - github.com/aristanetworks/goarista v0.0.0-20190607111240-52c2a7864a08 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.2.2 // indirect github.com/bombsimon/wsl/v2 v2.0.0 // indirect - github.com/btcsuite/btcd v0.21.0-beta // indirect + github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect + github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cockroachdb/errors v1.9.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/pebble v0.0.0-20230302152029-717cbce0c2e3 // indirect + github.com/cockroachdb/redact v1.1.3 // indirect github.com/containerd/cgroups v1.0.4 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect + github.com/deckarep/golang-set/v2 v2.1.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/dgraph-io/badger v1.6.2 // indirect github.com/dgraph-io/ristretto v0.0.3 // indirect @@ -103,8 +109,10 @@ require ( github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect + github.com/getsentry/sentry-go v0.18.0 // indirect github.com/go-critic/go-critic v0.4.0 // indirect github.com/go-lintpack/lintpack v0.5.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-stack/stack v1.8.1 // indirect github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect github.com/go-toolsmith/astcast v1.0.0 // indirect @@ -142,6 +150,9 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/holiman/big v0.0.0-20221017200358-a027dc42d04e // indirect + github.com/holiman/bloomfilter/v2 v2.0.3 // indirect + github.com/holiman/uint256 v1.2.1 // indirect github.com/huin/goupnp v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/ipfs/go-cid v0.3.2 // indirect @@ -154,13 +165,12 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect - github.com/jmespath/go-jmespath v0.3.0 // indirect - github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kisielk/gotool v1.0.0 // indirect - github.com/klauspost/compress v1.15.12 // indirect + github.com/klauspost/compress v1.16.0 // indirect github.com/klauspost/cpuid/v2 v2.2.1 // indirect github.com/koron/go-ssdp v0.0.3 // indirect - github.com/kr/pretty v0.3.0 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect @@ -183,7 +193,7 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.16 // indirect github.com/mattn/go-pointer v0.0.1 // indirect - github.com/mattn/go-runewidth v0.0.4 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/miekg/dns v1.1.50 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect @@ -204,7 +214,6 @@ require ( github.com/multiformats/go-multistream v0.3.3 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d // indirect - github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c // indirect github.com/onsi/ginkgo/v2 v2.5.1 // indirect github.com/opencontainers/runtime-spec v1.0.2 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect @@ -217,42 +226,44 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect - github.com/prometheus/tsdb v0.7.1 // indirect + github.com/prometheus/common v0.41.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/tsdb v0.10.0 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect - github.com/rogpeppe/go-internal v1.6.1 // indirect + github.com/rivo/uniseg v0.4.4 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d // indirect - github.com/sirupsen/logrus v1.8.1 // indirect + github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/sirupsen/logrus v1.9.0 // indirect github.com/sourcegraph/go-diff v0.5.1 // indirect github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.9.2 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect - github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 // indirect - github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 // indirect + github.com/status-im/keycard-go v0.2.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/subosito/gotenv v1.4.1 // indirect github.com/tikv/pd/client v0.0.0-20220216070739-26c668271201 // indirect github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect github.com/tommy-muehle/go-mnd v1.1.1 // indirect github.com/torquem-ch/mdbx-go v0.27.0 // indirect - github.com/tyler-smith/go-bip39 v1.0.2 // indirect + github.com/tyler-smith/go-bip39 v1.1.0 // indirect github.com/ultraware/funlen v0.0.2 // indirect github.com/ultraware/whitespace v0.0.4 // indirect github.com/uudashr/gocognit v1.0.1 // indirect github.com/valyala/fastrand v1.1.0 // indirect github.com/valyala/histogram v1.2.0 // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect - github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opencensus.io v0.24.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/dig v1.15.0 // indirect go.uber.org/fx v1.18.2 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect + golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 // indirect golang.org/x/mod v0.7.0 // indirect golang.org/x/term v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect @@ -268,4 +279,4 @@ require ( sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 // indirect ) -replace github.com/ethereum/go-ethereum => github.com/ethereum/go-ethereum v1.9.9 +replace github.com/ethereum/go-ethereum => github.com/ethereum/go-ethereum v1.11.2 diff --git a/go.sum b/go.sum index 46f3d441b0..81898e40b2 100644 --- a/go.sum +++ b/go.sum @@ -3,12 +3,14 @@ cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= @@ -19,14 +21,20 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= +cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -37,49 +45,57 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= -github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= -github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= +github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= +github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= +github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= +github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= +github.com/CloudyKit/jet/v6 v6.1.0/go.mod h1:d3ypHeIRNo2+XyqnGA8s+aphtcVpjP5hPwP/Lzo7Ro4= +github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= +github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= +github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= +github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= -github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us= github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= +github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= +github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06/go.mod h1:7erjKLwalezA0k99cWs5L11HWOAPNjdUZ6RxH1BXbbM= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= -github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw= -github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= +github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= +github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= +github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= github.com/VictoriaMetrics/metrics v1.23.0 h1:WzfqyzCaxUZip+OBbg1+lV33WChDSu4ssYII3nxtpeA= github.com/VictoriaMetrics/metrics v1.23.0/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc= github.com/Workiva/go-datastructures v1.0.50 h1:slDmfW6KCHcC7U+LP3DDBbm4fqTwZGn1beOFPfGaLvo= github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= github.com/Zilliqa/gozilliqa-sdk v1.2.1-0.20201201074141-dd0ecada1be6/go.mod h1:eSYp2T6f0apnuW8TzhV3f6Aff2SE8Dwio++U4ha4yEM= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -88,13 +104,27 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc= github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= -github.com/aristanetworks/goarista v0.0.0-20190607111240-52c2a7864a08 h1:UxoB3EYChE92EDNqRCS5vuE2ta4L/oKpeFaCK73KGvI= -github.com/aristanetworks/goarista v0.0.0-20190607111240-52c2a7864a08/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= +github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= +github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y= +github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2/go.mod h1:3hGg3PpiEjHnrkrlasTfxFqUsZ2GCk/fMUn4CbKgSkM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2/go.mod h1:45MfaXZ0cNbeuT0KQ1XJylq8A6+OpVV2E5kvY/Kq+u8= +github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1/go.mod h1:rLiOUrPLW/Er5kRcQ7NkwbjlijluLsrIbu/iyl35RO4= +github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0= +github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM= +github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/aws/aws-sdk-go v1.34.0 h1:brux2dRrlwCF5JhTL7MUT3WUwo9zfDHZZp3+g3Mvlmo= github.com/aws/aws-sdk-go v1.34.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= @@ -106,18 +136,27 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.2.2 h1:J5gbX05GpMdBjCvQ9MteIg2KKDExr7DrgK+Yc15FvIk= github.com/bits-and-blooms/bitset v1.2.2/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bombsimon/wsl v1.2.5/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM= github.com/bombsimon/wsl/v2 v2.0.0 h1:+Vjcn+/T5lSrO8Bjzhk4v14Un/2UyCA1E3V5j9nwTkQ= github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= -github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/btcsuite/btcd v0.0.0-20190315201642-aa6e0f35703c/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= -github.com/btcsuite/btcd v0.21.0-beta h1:At9hIZdJW0s9E/fAz28nrz6AmcNlSVucCH796ZteX1M= github.com/btcsuite/btcd v0.21.0-beta/go.mod h1:ZSWyehm27aAuS9bvkATT+Xte3hjHZ+MRgMY/8NJ7K94= +github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= +github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= +github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 h1:KdUfX2zKommPRa+PD0sWZUyXe9w277ABlgELO7H04IM= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= @@ -132,6 +171,7 @@ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b h1:6+ZFm0flnudZzdSE0JxlhR2hKnGPcNB35BjQf4RYQDY= github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= @@ -141,17 +181,17 @@ github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU= github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= +github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -159,8 +199,29 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= +github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= +github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= +github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811/go.mod h1:Nb5lgvnQ2+oGlE/EyZy4+2/CxRh9KfvCXnag1vtpxVM= +github.com/cockroachdb/pebble v0.0.0-20230302152029-717cbce0c2e3 h1:S4re5MXHfznkOlgkgUfh9ptgaG2esdH95IuJWwP0fM0= +github.com/cockroachdb/pebble v0.0.0-20230302152029-717cbce0c2e3/go.mod h1:9lRMC4XN3/BLPtIp6kAKwIaHu369NOf2rMucPzipz50= +github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= +github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/coinbase/rosetta-sdk-go v0.7.0 h1:lmTO/JEpCvZgpbkOITL95rA80CPKb5CtMzLaqF2mCNg= github.com/coinbase/rosetta-sdk-go v0.7.0/go.mod h1:7nD3oBPIiHqhRprqvMgPoGxe/nyq3yftRmpsy29coWE= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.9.1-0.20230105202408-1a7a29904a7c/go.mod h1:CkbdF9hbRidRJYMRzmfX8TMOr95I2pYXRHF18MzRrvA= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= @@ -176,49 +237,75 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= -github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= -github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ= -github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4= +github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= +github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI= +github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= +github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= +github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI= github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/djherbis/atime v1.1.0/go.mod h1:28OF6Y8s3NQWwacXc5eZTsEsiMzp7LF8MbXE+XJPdBE= +github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= +github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/docker/docker v1.6.2/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= +github.com/dop251/goja v0.0.0-20230122112309-96b1610dd4f7 h1:kgvzE5wLsLa7XKfV85VZl40QXaMCaeFtHpPwJ8fhotY= +github.com/dop251/goja v0.0.0-20230122112309-96b1610dd4f7/go.mod h1:yRkwfj0CBpOGre+TwBsqPV0IH0Pk73e4PXJOeNDboGs= +github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= +github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= -github.com/elastic/gosigar v0.8.1-0.20180330100440-37f05ff46ffa/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= @@ -230,34 +317,59 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.9.9 h1:jnoBvjH8aMH++iH14XmiJdAsnRcmZUM+B5fsnEZBVE0= -github.com/ethereum/go-ethereum v1.9.9/go.mod h1:a9TqabFudpDu1nucId+k9S8R9whYaHnGBLKFouA5EAo= -github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/ethereum/go-ethereum v1.11.2 h1:z/luyejbevDCAMUUiu0rc80dxJxOnpoG58k5o0tSawc= +github.com/ethereum/go-ethereum v1.11.2/go.mod h1:DuefStAgaxoaYGLR0FueVcVbehmn5n9QUcVrMCuOvuc= +github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= -github.com/fjl/memsize v0.0.0-20180929194037-2a09253e352a h1:1znxn4+q2MrEdTk1eCk6KIV3muTYVclBIB6CTVR/zBc= -github.com/fjl/memsize v0.0.0-20180929194037-2a09253e352a/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY= +github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= +github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= +github.com/flosch/pongo2/v4 v4.0.2/go.mod h1:B5ObFANs/36VwxxlgKpdchIJHMvHB562PW+BWPhwZD8= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILDlzrGEckF6HKjXe48EgsY/l7K7vhY4MW8= +github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/gballet/go-verkle v0.0.0-20220902153445-097bd83b7732/go.mod h1:o/XfIXWi4/GqbQirfRm5uTbXMG5NpqxkxblnbZ+QM9I= +github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= +github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= +github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= +github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= +github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= github.com/go-critic/go-critic v0.4.0 h1:sXD3pix0wDemuPuSlrXpJNNYXlUiKiysLrtPVQmxkzI= github.com/go-critic/go-critic v0.4.0/go.mod h1:7/14rZGnZbY6E38VEGk2kVhoq6itzc1E68facVDK23g= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -266,6 +378,7 @@ github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0= github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -274,9 +387,22 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= +github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= @@ -306,6 +432,14 @@ github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2X github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= +github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= @@ -313,11 +447,22 @@ github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v4 v4.3.0 h1:kHL1vqdqWNfATmA0FNMdmZNMyZI1U6O31X4rlIPoBog= +github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -332,11 +477,11 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= @@ -350,10 +495,13 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0= @@ -375,6 +523,7 @@ github.com/golangci/golangci-lint v1.22.2 h1:iaihss3Tf6NvZVjun3lHimKSgofPV1+FqE/ github.com/golangci/golangci-lint v1.22.2/go.mod h1:2Bj42k6hPQFTRxkDb7S3TQ+EsnumZXOmIYNqlQrp0FI= github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI= github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= +github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA= github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA= @@ -387,10 +536,12 @@ github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 h1:HVfrLniijszjS1 github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys= github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= +github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -402,10 +553,15 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= @@ -422,10 +578,13 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20221203041831-ce31453925ec h1:fR20TYVVwhK4O7r7y+McjRYyaTH6/vjwJOajE+XhlzM= github.com/google/pprof v0.0.0-20221203041831-ce31453925ec/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -436,18 +595,21 @@ github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE0 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw= github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= -github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= @@ -466,32 +628,68 @@ github.com/harmony-one/taggedrlp v0.1.4 h1:RZ+qy0VCzT+d/mTfq23gH3an5tSvxOhg6AddL github.com/harmony-one/taggedrlp v0.1.4/go.mod h1:osO5TRXLKdgCP+oj2J9qfqhywMOOA+4nP5q+o8nDSYA= github.com/harmony-one/vdf v0.0.0-20190924175951-620379da8849 h1:rMY4jLAen3pMTq9KO7kSXzuMaicnOHP5n1MgpA1T6G4= github.com/harmony-one/vdf v0.0.0-20190924175951-620379da8849/go.mod h1:EgNU7X5HLNBBho+OqCm1A1NrpD6xb1SHfi9pMCYaKKw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= +github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru v0.0.0-20160813221303-0a025b7e63ad/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/holiman/big v0.0.0-20221017200358-a027dc42d04e h1:pIYdhNkDh+YENVNi3gto8n9hAmRxKxoar0iE6BLucjw= +github.com/holiman/big v0.0.0-20221017200358-a027dc42d04e/go.mod h1:j9cQbcqHQujT0oKJ38PylVfqohClLr3CvDC+Qcg+lhU= +github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= +github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= +github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/holiman/uint256 v1.2.1 h1:XRtyuda/zw2l+Bq/38n5XUoEF72aSOu/77Thd9pPp2o= +github.com/holiman/uint256 v1.2.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v0.0.0-20161224104101-679507af18f3/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= +github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= +github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= +github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= +github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= +github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= +github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= +github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= +github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= +github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= +github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= @@ -512,6 +710,15 @@ github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscw github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= +github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= +github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= +github.com/iris-contrib/httpexpect/v2 v2.3.1/go.mod h1:ICTf89VBKSD3KB0fsyyHviKF8G8hyepP0dOXJPWz3T0= +github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= +github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= +github.com/iris-contrib/jade v1.1.4/go.mod h1:EDqR+ur9piDl6DUgs6qRrlfzmlx/D5UybogqrXvJTBE= +github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= +github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= +github.com/iris-contrib/schema v0.0.6/go.mod h1:iYszG0IOsuIsfzjymw1kMzTL8YQcCWlm65f3wX8J5iA= github.com/ipld/go-ipld-prime v0.19.0 h1:5axC7rJmPc17Emw6TelxGwnzALk0PdupZ2oj2roDj04= github.com/ipld/go-ipld-prime v0.19.0/go.mod h1:Q9j3BaVXwaA3o5JUDNvptDDr/x8+F7FG6XJ8WI3ILg4= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= @@ -522,28 +729,58 @@ github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABo github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw= -github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kataras/blocks v0.0.6/go.mod h1:UK+Iwk0Oxpc0GdoJja7sEildotAUKK1LYeYcVF0COWc= +github.com/kataras/blocks v0.0.7/go.mod h1:UJIU97CluDo0f+zEjbnbkeMRlvYORtmc1304EeyXf4I= +github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= +github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= +github.com/kataras/golog v0.1.7/go.mod h1:jOSQ+C5fUqsNSwurB/oAHq1IFSb0KI3l6GMa7xB6dZA= +github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= +github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= +github.com/kataras/iris/v12 v12.2.0-beta5/go.mod h1:q26aoWJ0Knx/00iPKg5iizDK7oQQSPjbD8np0XDh6dc= +github.com/kataras/jwt v0.1.8/go.mod h1:Q5j2IkcIHnfwy+oNY3TVWuEBJNw0ADgCcXK9CaZwV4o= +github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= +github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= +github.com/kataras/neffos v0.0.20/go.mod h1:srdvC/Uo8mgrApWW0AYtiiLgMbyNPf69qPsd2FhE6MQ= +github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= +github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= +github.com/kataras/pio v0.0.10/go.mod h1:gS3ui9xSD+lAUpbYnjOGiQyY7sUMJO+EHpiRzhtZ5no= +github.com/kataras/pio v0.0.11/go.mod h1:38hH6SWH6m4DKSYmRhlrCJ5WItwWgCVrTNU62XZyUvI= +github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= +github.com/kataras/sitemap v0.0.6/go.mod h1:dW4dOCNs896OR1HmG+dMLdT7JjDk7mYBzoIRwuj5jA4= +github.com/kataras/tunnel v0.0.4/go.mod h1:9FkU4LaeifdMWqZu7o20ojmW4B7hdhv2CMLwfnHGpYw= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -552,14 +789,27 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM= -github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= +github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.1 h1:U33DW0aiEj633gHYw3LoDNfkDiYnE5Q8M/TKJn2f2jI= github.com/klauspost/cpuid/v2 v2.2.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= +github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -570,19 +820,30 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= +github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= +github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= +github.com/labstack/echo/v4 v4.9.0/go.mod h1:xkCDAdFCIf8jsFQ5NnbK7oqaF/yU1A1X20Ltm0OvSks= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/labstack/gommon v0.3.1/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/ledgerwatch/erigon-lib v0.0.0-20221218022306-0f8fdd40c2db h1:wV9YkkYQArbUdTdlPxXi5BW6H9ovYbyUT8Af7foetvQ= github.com/ledgerwatch/erigon-lib v0.0.0-20221218022306-0f8fdd40c2db/go.mod h1:5GCPOzxAshLF7f0wrMZu2Bdq0qqIiMcIubM9n+25gGo= github.com/ledgerwatch/log/v3 v3.6.0 h1:JBUSK1epPyutUrz7KYDTcJtQLEHnehECRpKbM1ugy5M= github.com/ledgerwatch/log/v3 v3.6.0/go.mod h1:L+Sp+ma/h205EdCjviZECjGEvYUYEyXSdiuHNZzg+xQ= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= @@ -623,12 +884,18 @@ github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQ github.com/lucas-clemente/quic-go v0.31.0 h1:MfNp3fk0wjWRajw6quMFA3ap1AVtlU+2mtwmbVogB2M= github.com/lucas-clemente/quic-go v0.31.0/go.mod h1:0wFbizLgYzqHqtlyxyCaJKlE7bYgE6JQ+54TLd/Dq2g= github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailgun/raymond/v2 v2.0.46/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/marten-seemann/qpack v0.3.0 h1:UiWstOgT8+znlkDPOg2+3rIuYXJ2CnGDkGUXN6ki6hE= github.com/marten-seemann/qtls-go1-18 v0.1.3 h1:R4H2Ks8P6pAtUagjFty2p7BVHn3XiwDAl7TTQf5h7TI= github.com/marten-seemann/qtls-go1-18 v0.1.3/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= @@ -639,16 +906,23 @@ github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs github.com/marten-seemann/webtransport-go v0.4.1 h1:8Ir7OoAvtp79yxQpa3foTKIPuoH+0eKpisHObJyu9Sk= github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE= github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= -github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= +github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= @@ -656,13 +930,25 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= -github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= +github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= +github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= +github.com/mediocregopher/radix/v3 v3.8.0/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50= +github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= @@ -673,16 +959,28 @@ github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKo github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc= github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= +github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -690,11 +988,14 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= +github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= @@ -729,22 +1030,37 @@ github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k= +github.com/nats-io/jwt/v2 v2.3.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k= +github.com/nats-io/nats-server/v2 v2.8.4/go.mod h1:8zZa+Al3WsESfmgSs98Fi06dRWLH5Bnq90m5bKD/eT4= +github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nats.go v1.15.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= +github.com/nats-io/nats.go v1.16.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= +github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E= github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/neilotoole/errgroup v0.1.5/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c h1:1RHs3tNxjXGHeul8z2t6H2N2TlAqpKe5yryJztRx4Jk= -github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= @@ -762,23 +1078,29 @@ github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5h github.com/onsi/gomega v1.24.0 h1:+0glovB9Jd6z3VR+ScSwQqXVTIfJcGA9UBM8yzQxhqg= github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= -github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= +github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 h1:HVl5539r48eA+uDuX/ziBmQCxzT1pGrzWbKuXT46Bq0= github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= @@ -795,13 +1117,18 @@ github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898/go.mod h1:IOdRDPLy github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee h1:VO2t6IBpfvW34TdtD/G10VvnGqjLic1jzOuHjUb5VqM= github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e h1:ZOcivgkkFRnjfoTcGsDq3UQYiBmekwLA+qg0OjyB/ls= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -810,6 +1137,7 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.0/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= @@ -817,17 +1145,21 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1: github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= +github.com/prometheus/common v0.41.0 h1:npo01n6vUlRViIj5fgwiK8vlNIh8bnoxqh3gypKsyAw= +github.com/prometheus/common v0.41.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -835,47 +1167,67 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= +github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= +github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8= github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM= -github.com/robertkrimen/otto v0.0.0-20170205013659-6a77b7cbc37d/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.18.0 h1:CbAm3kP2Tptby1i9sYy2MGRg0uxIN9cyDb59Ys7W8z8= github.com/rs/zerolog v1.18.0/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d h1:BzRvVq1EHuIjxpijCEKpAxzKUUMurOQ4sknehIATRh8= github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= +github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= +github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil/v3 v3.22.8/go.mod h1:s648gW4IywYzUfE/KjXxUsqrqx/T2xO5VqOXxONeRfI= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= -github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM= github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636 h1:aSISeOcal5irEhJd1M+IrApc0PdcN7e7Aj4yuEnOrfQ= +github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc= github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= @@ -886,6 +1238,7 @@ github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9A github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= @@ -895,17 +1248,22 @@ github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1l github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= +github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/assertions v1.13.0 h1:Dx1kYM01xsSqKPno3aqLnrwac2LetPvN23diwyr69Qs= +github.com/smartystreets/assertions v1.13.0/go.mod h1:wDmR7qL282YbGsPy6H/yAsesrxfxaaSlJazyFLYVFx8= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs= @@ -914,17 +1272,22 @@ github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= -github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -934,20 +1297,19 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= -github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= -github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= -github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE= -github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= -github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639dk2kqnCPPv+wNjq7Xb6EfUxe/oX0/NM= -github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= +github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= +github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -955,16 +1317,22 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= -github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= +github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= +github.com/tdewolff/minify/v2 v2.12.1/go.mod h1:p5pwbvNs1ghbFED/ZW1towGsnnWwzvM8iz8l0eURi9g= +github.com/tdewolff/minify/v2 v2.12.4/go.mod h1:h+SRvSIX3kwgwTFOpSckvSxgax3uy8kZTSF1Ojrr3bk= +github.com/tdewolff/parse/v2 v2.6.3/go.mod h1:woz0cgbLwFdtbjJu8PIKxhW05KplTFQkOdX78o+Jgrs= +github.com/tdewolff/parse/v2 v2.6.4/go.mod h1:woz0cgbLwFdtbjJu8PIKxhW05KplTFQkOdX78o+Jgrs= +github.com/tdewolff/test v1.0.7/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= github.com/tidwall/gjson v1.6.7/go.mod h1:zeFuBCIqD4sN/gmqBzZ4j7Jd6UcA2Fc56x7QFsv+8fI= github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -975,57 +1343,99 @@ github.com/tikv/pd/client v0.0.0-20220216070739-26c668271201 h1:7h/Oi4Zw6eGCeXh4 github.com/tikv/pd/client v0.0.0-20220216070739-26c668271201/go.mod h1:fEvI5fhAuJn1Fn87VJF8ByE9Vc16EzWGoePZB21/nL8= github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q= github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= +github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= +github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= +github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tommy-muehle/go-mnd v1.1.1 h1:4D0wuPKjOTiK2garzuPGGvm4zZ/wLYDOH8TJSABC7KU= github.com/tommy-muehle/go-mnd v1.1.1/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig= github.com/torquem-ch/mdbx-go v0.27.0 h1:FquhRvKL2zweMdk1R6UdOx3h6DiHgJ0+P9yQvSouURI= github.com/torquem-ch/mdbx-go v0.27.0/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E= github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= -github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= -github.com/tyler-smith/go-bip39 v1.0.2 h1:+t3w+KwLXO6154GNJY+qUtIxLTmFjfUmpguQT1OlOT8= github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= +github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= +github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo= github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg= github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa h1:5SqCsI/2Qya2bCzK15ozrqo2sZxkh0FHynJZOTVoV6Q= +github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= +github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= github.com/uudashr/gocognit v1.0.1 h1:MoG2fZ0b/Eo7NXoIwCVFLG5JED3qgQz5/NEE+rOsjPs= github.com/uudashr/gocognit v1.0.1/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= +github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/fasthttp v1.40.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ= github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY= github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vmihailenco/msgpack/v5 v5.1.4/go.mod h1:C5gboKD0TJPqWDTVTtrQNfRbiBwHZGo8UTqP/9/XvLI= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= -github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk= -github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= +github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= +github.com/yosssi/ace v0.0.5/go.mod h1:ALfIzm2vT7t5ZE7uoIZqF3TQ7SAOyupFZnkrF5id+K0= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/api/v3 v3.5.2/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/pkg/v3 v3.5.2/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.etcd.io/etcd/client/v3 v3.5.2/go.mod h1:kOOaWFFgHygyT0WlSmL8TJiXmMysO/nNUlEsSsN6W4o= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -1034,6 +1444,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -1080,6 +1491,7 @@ go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1 golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -1088,19 +1500,39 @@ golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= -golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= @@ -1110,8 +1542,12 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20221205204356-47842c84f3db h1:D/cFflL63o2KSLJIwjlcIt8PR064j/xsmdEJL/YvY/o= -golang.org/x/exp v0.0.0-20221205204356-47842c84f3db/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/exp v0.0.0-20220426173459-3bcf042a4bf5/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 h1:Jvc7gsqn21cJHCmAWx0LiimpP18LZmUxkT5Mp7EZ1mI= +golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1138,22 +1574,30 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1163,6 +1607,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1182,19 +1627,34 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1208,8 +1668,12 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1222,17 +1686,22 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1246,9 +1715,10 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1257,6 +1727,7 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1277,17 +1748,28 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1296,16 +1778,40 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1317,12 +1823,19 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE= golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1332,8 +1845,10 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190221204921-83362c3779f5/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1341,6 +1856,7 @@ golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -1362,14 +1878,17 @@ golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113232020-e2727e816f5a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191126055441-b0650ceb63d9/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1400,14 +1919,26 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= @@ -1430,6 +1961,9 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1439,6 +1973,7 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -1448,6 +1983,7 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -1456,6 +1992,7 @@ google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= @@ -1481,10 +2018,17 @@ google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e h1:S9GbmC1iCgvbLyAokVCwiO6tVIrU9Y7c5oMx1V/ki/Y= google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1508,6 +2052,7 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= @@ -1524,33 +2069,38 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 h1:a6cXbcDDUkSBlpnkWV1bJ+vv3mOgQEltEJ2rPxroVu0= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= -gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1562,11 +2112,11 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1580,6 +2130,7 @@ honnef.co/go/tools v0.0.1-2020.1.5 h1:nI5egYTGJakVyOryqLs1cQO5dO0ksin5XXs2pspk75 honnef.co/go/tools v0.0.1-2020.1.5/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE= mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I= mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo= @@ -1587,8 +2138,10 @@ mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jC mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4= mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 h1:JPJh2pk3+X4lXAkZIk2RuE/7/FoK9maXw+TNPJhVS/c= diff --git a/hmy/blockchain.go b/hmy/blockchain.go index f72be5d830..2ee030986d 100644 --- a/hmy/blockchain.go +++ b/hmy/blockchain.go @@ -362,11 +362,11 @@ func (hmy *Harmony) GetLogs(ctx context.Context, blockHash common.Hash, isEth bo return nil, errors.New("Missing block data") } txns := block.Transactions() - for i, _ := range receipts { + for i := range receipts { if i < len(txns) { ethHash := txns[i].ConvertToEth().Hash() receipts[i].TxHash = ethHash - for j, _ := range receipts[i].Logs { + for j := range receipts[i].Logs { // Override log txHash with receipt's receipts[i].Logs[j].TxHash = ethHash } diff --git a/hmy/tracer.go b/hmy/tracer.go index 699f784297..2529e7a2f6 100644 --- a/hmy/tracer.go +++ b/hmy/tracer.go @@ -122,7 +122,7 @@ func (hmy *Harmony) TraceChain(ctx context.Context, start, end *types.Block, con } } - statedb, err := state.New(start.Root(), database) + statedb, err := state.New(start.Root(), database, nil) if err != nil { // If the starting state is missing, allow some number of blocks to be executed reexec := defaultTraceReexec @@ -135,7 +135,7 @@ func (hmy *Harmony) TraceChain(ctx context.Context, start, end *types.Block, con if start == nil { break } - if statedb, err = state.New(start.Root(), database); err == nil { + if statedb, err = state.New(start.Root(), database, nil); err == nil { break } } @@ -644,7 +644,7 @@ func (hmy *Harmony) ComputeStateDB(block *types.Block, reexec uint64) (*state.DB if block == nil { break } - if statedb, err = state.New(block.Root(), database); err == nil { + if statedb, err = state.New(block.Root(), database, nil); err == nil { break } } diff --git a/internal/chain/engine_test.go b/internal/chain/engine_test.go index 463a0afdb6..07810d83f2 100644 --- a/internal/chain/engine_test.go +++ b/internal/chain/engine_test.go @@ -228,7 +228,7 @@ func (maker *shardSlotMaker) makeSlot() shard.Slot { func makeTestStateDB() *state.DB { db := state.NewDatabase(rawdb.NewMemoryDatabase()) - sdb, err := state.New(common.Hash{}, db) + sdb, err := state.New(common.Hash{}, db, nil) if err != nil { panic(err) } diff --git a/internal/shardchain/dbfactory.go b/internal/shardchain/dbfactory.go index 762ac54106..6d8657e787 100644 --- a/internal/shardchain/dbfactory.go +++ b/internal/shardchain/dbfactory.go @@ -9,7 +9,7 @@ import ( "github.com/harmony-one/harmony/internal/shardchain/leveldb_shard" "github.com/harmony-one/harmony/internal/shardchain/local_cache" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/harmony-one/harmony/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" ) @@ -34,7 +34,7 @@ type LDBFactory struct { // NewChainDB returns a new LDB for the blockchain for given shard. func (f *LDBFactory) NewChainDB(shardID uint32) (ethdb.Database, error) { dir := path.Join(f.RootDir, fmt.Sprintf("%s_%d", LDBDirPrefix, shardID)) - return rawdb.NewLevelDBDatabase(dir, 256, 1024, "") + return rawdb.NewLevelDBDatabase(dir, 256, 1024, "", false) } // MemDBFactory is a memory-backed blockchain database factory. diff --git a/internal/shardchain/leveldb_shard/shard.go b/internal/shardchain/leveldb_shard/shard.go index 3f2339c427..607996ff29 100644 --- a/internal/shardchain/leveldb_shard/shard.go +++ b/internal/shardchain/leveldb_shard/shard.go @@ -96,6 +96,14 @@ func NewLeveldbShard(savePath string, diskCount int, diskShards int) (shard *Lev return shard, err } +func (l *LeveldbShard) NewBatchWithSize(size int) ethdb.Batch { + return nil +} + +func (l *LeveldbShard) NewSnapshot() (ethdb.Snapshot, error) { + return nil, nil +} + func (l *LeveldbShard) mapDB(key []byte) *leveldb.DB { return l.dbs[mapDBIndex(key, l.dbCount)] } @@ -128,7 +136,7 @@ func (l *LeveldbShard) NewBatch() ethdb.Batch { // NewIterator creates a binary-alphabetical iterator over the entire keyspace // contained within the key-value database. -func (l *LeveldbShard) NewIterator() ethdb.Iterator { +func (l *LeveldbShard) NewIterator(prefix []byte, start []byte) ethdb.Iterator { return l.iterator(nil) } diff --git a/internal/tikv/common/tikv_store.go b/internal/tikv/common/tikv_store.go index 2ad342ad8f..2a60972b84 100644 --- a/internal/tikv/common/tikv_store.go +++ b/internal/tikv/common/tikv_store.go @@ -1,6 +1,7 @@ package common import ( + "errors" "io" "github.com/ethereum/go-ethereum/ethdb" @@ -23,7 +24,7 @@ type TiKVStoreWrapper struct { TiKVStore } -func (t *TiKVStoreWrapper) NewIterator() ethdb.Iterator { +func (t *TiKVStoreWrapper) NewIterator(prefix []byte, start []byte) ethdb.Iterator { return t.TiKVStore.NewIterator(nil, nil) } @@ -36,6 +37,10 @@ func (t *TiKVStoreWrapper) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator { return t.TiKVStore.NewIterator(bytesPrefix.Start, bytesPrefix.Limit) } +func (t *TiKVStoreWrapper) NewSnapshot() (ethdb.Snapshot, error) { + return nil, errors.New("not supported") +} + func ToEthKeyValueStore(store TiKVStore) ethdb.KeyValueStore { return &TiKVStoreWrapper{TiKVStore: store} } diff --git a/internal/tikv/prefix/prefix_database.go b/internal/tikv/prefix/prefix_database.go index 7bfe9d9451..7a04a0ab35 100644 --- a/internal/tikv/prefix/prefix_database.go +++ b/internal/tikv/prefix/prefix_database.go @@ -21,6 +21,15 @@ func NewPrefixDatabase(prefix []byte, db common.TiKVStore) *PrefixDatabase { } } +func (p *PrefixDatabase) AncientDatadir() (string, error) { + return "", nil +} + +// NewBatchWithSize creates a write-only database batch with pre-allocated buffer. +func (p *PrefixDatabase) NewBatchWithSize(size int) ethdb.Batch { + return nil +} + // makeKey use to create a key with prefix, keysPool can reduce gc pressure func (p *PrefixDatabase) makeKey(keys []byte) []byte { prefixLen := len(p.prefix) diff --git a/internal/tikv/remote/remote_database.go b/internal/tikv/remote/remote_database.go index 915e123754..cf33c719d8 100644 --- a/internal/tikv/remote/remote_database.go +++ b/internal/tikv/remote/remote_database.go @@ -39,6 +39,14 @@ func (d *RemoteDatabase) ReadOnly() { d.readOnly = true } +func (d *RemoteDatabase) AncientDatadir() (string, error) { + return "", nil +} + +func (d *RemoteDatabase) NewBatchWithSize(size int) ethdb.Batch { + return nil +} + // Has retrieves if a key is present in the key-value data store. func (d *RemoteDatabase) Has(key []byte) (bool, error) { data, err := d.Get(key) diff --git a/internal/tikv/statedb_cache/statedb_cache_database.go b/internal/tikv/statedb_cache/statedb_cache_database.go index 9757298130..b4c95b9bc9 100644 --- a/internal/tikv/statedb_cache/statedb_cache_database.go +++ b/internal/tikv/statedb_cache/statedb_cache_database.go @@ -139,6 +139,14 @@ func NewStateDBCacheDatabase(remoteDB common.TiKVStore, config StateDBCacheConfi return db, nil } +func (c *StateDBCacheDatabase) AncientDatadir() (string, error) { + return "", nil +} + +func (c *StateDBCacheDatabase) NewBatchWithSize(size int) ethdb.Batch { + return nil +} + // Has retrieves if a key is present in the key-value data store. func (c *StateDBCacheDatabase) Has(key []byte) (bool, error) { return c.remoteDB.Has(key) diff --git a/rpc/blockchain.go b/rpc/blockchain.go index 46e495944d..ae588e7500 100644 --- a/rpc/blockchain.go +++ b/rpc/blockchain.go @@ -807,7 +807,10 @@ func (s *PublicBlockchainService) GetProof( return nil, err } - storageTrie := state.StorageTrie(address) + storageTrie, errTr := state.StorageTrie(address) + if errTr != nil { + return + } storageHash := types.EmptyRootHash codeHash := state.GetCodeHash(address) storageProof := make([]StorageResult, len(storageKeys)) diff --git a/staking/slash/double-sign_test.go b/staking/slash/double-sign_test.go index 7dbd3ed707..670a99d7c6 100644 --- a/staking/slash/double-sign_test.go +++ b/staking/slash/double-sign_test.go @@ -1171,7 +1171,7 @@ func defaultTestStateDB() *state.DB { func makeTestStateDB() *state.DB { db := state.NewDatabase(rawdb.NewMemoryDatabase()) - sdb, err := state.New(common.Hash{}, db) + sdb, err := state.New(common.Hash{}, db, nil) if err != nil { panic(err) } diff --git a/test/chain/chain/chain_makers.go b/test/chain/chain/chain_makers.go index b4ca9ecd7c..a902fc8441 100644 --- a/test/chain/chain/chain_makers.go +++ b/test/chain/chain/chain_makers.go @@ -216,7 +216,7 @@ func GenerateChain( return nil, nil } for i := 0; i < n; i++ { - statedb, err := state.New(parent.Root(), state.NewDatabase(db)) + statedb, err := state.New(parent.Root(), state.NewDatabase(db), nil) if err != nil { panic(err) } diff --git a/test/chain/reward/main.go b/test/chain/reward/main.go index 3848697b66..2f79378e35 100644 --- a/test/chain/reward/main.go +++ b/test/chain/reward/main.go @@ -110,7 +110,7 @@ func main() { _ = genesis engine := chain.NewEngine() bc, _ := core.NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, engine, vm.Config{}) - statedb, _ := state.New(common2.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) + statedb, _ := state.New(common2.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) msg := createValidator() statedb.AddBalance(msg.ValidatorAddress, new(big.Int).Mul(big.NewInt(5e18), big.NewInt(2000))) validator, err := core.VerifyAndCreateValidatorFromMsg( From 47604587322de24255451899f1fd2dd2d49c1a9b Mon Sep 17 00:00:00 2001 From: Gheis Mohammadi Date: Sat, 1 Apr 2023 01:28:44 +0800 Subject: [PATCH 386/420] update all ethereum rawdb pointers to use harmony rawdb (#4395) --- cmd/harmony/dumpdb.go | 5 ++--- core/epochchain_test.go | 2 +- core/evm_test.go | 2 +- core/rawdb/accessors_chain_test.go | 18 ++++++++---------- core/rawdb/accessors_indexes_test.go | 6 ++---- core/rawdb/accessors_snapdb_test.go | 3 +-- core/staking_verifier_test.go | 2 +- core/state/managed_state_test.go | 2 +- core/tx_pool_test.go | 2 +- core/vm/contracts_write_test.go | 2 +- core/vm/gas_table_test.go | 2 +- core/vm/runtime/runtime.go | 2 +- core/vm/runtime/runtime_test.go | 2 +- hmy/bloombits.go | 3 +-- internal/chain/engine_test.go | 2 +- internal/shardchain/dbfactory_tikv.go | 2 +- node/worker/worker_test.go | 2 +- staking/slash/double-sign_test.go | 2 +- test/chain/main.go | 2 +- test/chain/reward/main.go | 2 +- 20 files changed, 29 insertions(+), 36 deletions(-) diff --git a/cmd/harmony/dumpdb.go b/cmd/harmony/dumpdb.go index 72e9de0300..5803359e5b 100644 --- a/cmd/harmony/dumpdb.go +++ b/cmd/harmony/dumpdb.go @@ -12,7 +12,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" - ethRawDB "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" @@ -367,12 +366,12 @@ func (db *KakashiDB) stateDataDump(block *types.Block) { func dumpMain(srcDBDir, destDBDir string, batchLimit int) { fmt.Println("===dumpMain===") - srcDB, err := ethRawDB.NewLevelDBDatabase(srcDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "", false) + srcDB, err := rawdb.NewLevelDBDatabase(srcDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "", false) if err != nil { fmt.Println("open src db error:", err) os.Exit(-1) } - destDB, err := ethRawDB.NewLevelDBDatabase(destDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "", false) + destDB, err := rawdb.NewLevelDBDatabase(destDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "", false) if err != nil { fmt.Println("open dest db error:", err) os.Exit(-1) diff --git a/core/epochchain_test.go b/core/epochchain_test.go index eec7bd6280..32f0dd530b 100644 --- a/core/epochchain_test.go +++ b/core/epochchain_test.go @@ -3,8 +3,8 @@ package core_test import ( "testing" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/vm" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/stretchr/testify/require" diff --git a/core/evm_test.go b/core/evm_test.go index 962dedd307..cab3f712cf 100644 --- a/core/evm_test.go +++ b/core/evm_test.go @@ -9,13 +9,13 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" bls_core "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/block" blockfactory "github.com/harmony-one/harmony/block/factory" "github.com/harmony-one/harmony/common/denominations" + "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index e9dea08824..6edf9f9ec7 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -21,8 +21,6 @@ import ( "math/big" "testing" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/block" @@ -34,7 +32,7 @@ import ( // Tests block header storage and retrieval operations. func TestHeaderStorage(t *testing.T) { - db := rawdb.NewMemoryDatabase() + db := NewMemoryDatabase() // Create a test header to move around the database and make sure it's really new header := blockfactory.NewTestHeader().With().Number(big.NewInt(42)).Extra([]byte("test header")).Header() @@ -67,7 +65,7 @@ func TestHeaderStorage(t *testing.T) { // Tests block body storage and retrieval operations. func TestBodyStorage(t *testing.T) { - db := rawdb.NewMemoryDatabase() + db := NewMemoryDatabase() // Create a test body to move around the database and make sure it's really new body := types.NewTestBody().With().Uncles([]*block.Header{blockfactory.NewTestHeader().With().Extra([]byte("test header")).Header()}).Body() @@ -105,7 +103,7 @@ func TestBodyStorage(t *testing.T) { // Tests block storage and retrieval operations. func TestBlockStorage(t *testing.T) { - db := rawdb.NewMemoryDatabase() + db := NewMemoryDatabase() // Create a test block to move around the database and make sure it's really new block := types.NewBlockWithHeader(blockfactory.NewTestHeader().With(). @@ -167,7 +165,7 @@ func TestBlockStorage(t *testing.T) { // Tests that partial block contents don't get reassembled into full blocks. func TestPartialBlockStorage(t *testing.T) { - db := rawdb.NewMemoryDatabase() + db := NewMemoryDatabase() block := types.NewBlockWithHeader(blockfactory.NewTestHeader().With(). Extra([]byte("test block")). TxHash(types.EmptyRootHash). @@ -200,7 +198,7 @@ func TestPartialBlockStorage(t *testing.T) { // Tests block total difficulty storage and retrieval operations. func TestTdStorage(t *testing.T) { - db := rawdb.NewMemoryDatabase() + db := NewMemoryDatabase() // Create a test TD to move around the database and make sure it's really new hash, td := common.Hash{}, big.NewInt(314) @@ -223,7 +221,7 @@ func TestTdStorage(t *testing.T) { // Tests that canonical numbers can be mapped to hashes and retrieved. func TestCanonicalMappingStorage(t *testing.T) { - db := rawdb.NewMemoryDatabase() + db := NewMemoryDatabase() // Create a test canonical number and assinged hash to move around hash, number := common.Hash{0: 0xff}, uint64(314) @@ -246,7 +244,7 @@ func TestCanonicalMappingStorage(t *testing.T) { // Tests that head headers and head blocks can be assigned, individually. func TestHeadStorage(t *testing.T) { - db := rawdb.NewMemoryDatabase() + db := NewMemoryDatabase() blockHead := types.NewBlockWithHeader(blockfactory.NewTestHeader().With().Extra([]byte("test block header")).Header()) blockFull := types.NewBlockWithHeader(blockfactory.NewTestHeader().With().Extra([]byte("test block full")).Header()) @@ -281,7 +279,7 @@ func TestHeadStorage(t *testing.T) { // Tests that receipts associated with a single block can be stored and retrieved. func TestBlockReceiptStorage(t *testing.T) { - db := rawdb.NewMemoryDatabase() + db := NewMemoryDatabase() receipt1 := &types.Receipt{ Status: types.ReceiptStatusFailed, diff --git a/core/rawdb/accessors_indexes_test.go b/core/rawdb/accessors_indexes_test.go index 77a66f63e0..41f54ac98a 100644 --- a/core/rawdb/accessors_indexes_test.go +++ b/core/rawdb/accessors_indexes_test.go @@ -20,8 +20,6 @@ import ( "math/big" "testing" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/harmony-one/harmony/crypto/bls" "github.com/ethereum/go-ethereum/common" @@ -42,7 +40,7 @@ var ( // Tests that positional lookup metadata can be stored and retrieved. func TestLookupStorage(t *testing.T) { - db := rawdb.NewMemoryDatabase() + db := NewMemoryDatabase() tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), 0, big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11}) tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), 0, big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22}) @@ -123,7 +121,7 @@ func TestLookupStorage(t *testing.T) { // Test that staking tx hash does not find a plain tx hash (and visa versa) within the same block func TestMixedLookupStorage(t *testing.T) { - db := rawdb.NewMemoryDatabase() + db := NewMemoryDatabase() tx := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), 0, big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11}) stx := sampleCreateValidatorStakingTxn() diff --git a/core/rawdb/accessors_snapdb_test.go b/core/rawdb/accessors_snapdb_test.go index e81a856b26..915a28e8ca 100644 --- a/core/rawdb/accessors_snapdb_test.go +++ b/core/rawdb/accessors_snapdb_test.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/ethereum/go-ethereum/common/hexutil" - ethRawDB "github.com/ethereum/go-ethereum/core/rawdb" blockfactory "github.com/harmony-one/harmony/block/factory" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" ) @@ -22,7 +21,7 @@ func TestSnapdbInfo(t *testing.T) { LastAccountKey: hexutil.MustDecode("0x1339383fd90ed804e28464763a13fafad66dd0f88434b8e5d8b410eb75a10331"), LastAccountStateKey: hexutil.MustDecode("0xa940a0bb9eca4f9d5eee3f4059f458bd4a05bb1d680c5f7c781b06bc43c20df6"), } - db := ethRawDB.NewMemoryDatabase() + db := NewMemoryDatabase() if err := WriteSnapdbInfo(db, src); err != nil { t.Fatal(err) } diff --git a/core/staking_verifier_test.go b/core/staking_verifier_test.go index eeb372889b..01d24cc8d4 100644 --- a/core/staking_verifier_test.go +++ b/core/staking_verifier_test.go @@ -9,7 +9,7 @@ import ( "github.com/harmony-one/harmony/internal/params" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/crypto/bls" diff --git a/core/state/managed_state_test.go b/core/state/managed_state_test.go index d1fbb906d5..81d316c3ab 100644 --- a/core/state/managed_state_test.go +++ b/core/state/managed_state_test.go @@ -19,7 +19,7 @@ package state import ( "testing" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/harmony-one/harmony/core/rawdb" "github.com/ethereum/go-ethereum/common" ) diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 5d11b50e9a..9e1bba296a 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -27,7 +27,7 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/crypto/bls" diff --git a/core/vm/contracts_write_test.go b/core/vm/contracts_write_test.go index c7269e6fff..479bf420fc 100644 --- a/core/vm/contracts_write_test.go +++ b/core/vm/contracts_write_test.go @@ -7,8 +7,8 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" + "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/params" diff --git a/core/vm/gas_table_test.go b/core/vm/gas_table_test.go index 4619ccd029..08e265c744 100644 --- a/core/vm/gas_table_test.go +++ b/core/vm/gas_table_test.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/state" ) diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index 69275fc58f..5595866daa 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -22,8 +22,8 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" + "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/internal/params" diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 09f2b6fc49..a4f86ed8b6 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -21,7 +21,7 @@ import ( "strings" "testing" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/harmony-one/harmony/core/rawdb" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" diff --git a/hmy/bloombits.go b/hmy/bloombits.go index aaef781cac..461372de23 100644 --- a/hmy/bloombits.go +++ b/hmy/bloombits.go @@ -23,7 +23,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/bitutil" "github.com/ethereum/go-ethereum/core/bloombits" - ethRawDB "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/harmony-one/harmony/block" "github.com/harmony-one/harmony/core" @@ -72,7 +71,7 @@ func NewBloomIndexer(db core.Chain, size, confirms uint64) *core.ChainIndexer { db: db.ChainDb(), size: size, } - table := ethRawDB.NewTable(db.ChainDb(), string(rawdb.BloomBitsIndexPrefix)) + table := rawdb.NewTable(db.ChainDb(), string(rawdb.BloomBitsIndexPrefix)) return core.NewChainIndexer(db, table, backend, size, confirms, bloomThrottling, "bloombits") } diff --git a/internal/chain/engine_test.go b/internal/chain/engine_test.go index 07810d83f2..5a842088c8 100644 --- a/internal/chain/engine_test.go +++ b/internal/chain/engine_test.go @@ -19,7 +19,7 @@ import ( types2 "github.com/harmony-one/harmony/staking/types" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/params" diff --git a/internal/shardchain/dbfactory_tikv.go b/internal/shardchain/dbfactory_tikv.go index 6651cd5c36..80b0117eda 100644 --- a/internal/shardchain/dbfactory_tikv.go +++ b/internal/shardchain/dbfactory_tikv.go @@ -5,7 +5,7 @@ import ( "io" "sync" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/internal/tikv" tikvCommon "github.com/harmony-one/harmony/internal/tikv/common" "github.com/harmony-one/harmony/internal/tikv/prefix" diff --git a/node/worker/worker_test.go b/node/worker/worker_test.go index e6ffdfddad..37d61816cc 100644 --- a/node/worker/worker_test.go +++ b/node/worker/worker_test.go @@ -7,7 +7,7 @@ import ( "github.com/harmony-one/harmony/core/state" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/harmony-one/harmony/core/rawdb" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" diff --git a/staking/slash/double-sign_test.go b/staking/slash/double-sign_test.go index 670a99d7c6..470a8ae1d8 100644 --- a/staking/slash/double-sign_test.go +++ b/staking/slash/double-sign_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/crypto/bls" diff --git a/test/chain/main.go b/test/chain/main.go index e78b519356..fd1a74e737 100644 --- a/test/chain/main.go +++ b/test/chain/main.go @@ -6,7 +6,7 @@ import ( "log" "math/big" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/harmony-one/harmony/core/rawdb" chain2 "github.com/harmony-one/harmony/test/chain/chain" "github.com/ethereum/go-ethereum/common" diff --git a/test/chain/reward/main.go b/test/chain/reward/main.go index 2f79378e35..5cfe9425a1 100644 --- a/test/chain/reward/main.go +++ b/test/chain/reward/main.go @@ -6,7 +6,7 @@ import ( "math/rand" "time" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/harmony-one/harmony/core/rawdb" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/crypto/bls" From 0a18d1650419aff98eb2a592638f5910dbe5d034 Mon Sep 17 00:00:00 2001 From: Konstantin <355847+Frozen@users.noreply.github.com> Date: Tue, 4 Apr 2023 23:46:07 -0300 Subject: [PATCH 387/420] Fix reduce node dependencies. (#4379) * Fix. * Fix. * Fix pool init. * Clean up. --- hmy/blockchain.go | 12 ++++- hmy/hmy.go | 17 ++++--- internal/registry/registry.go | 18 +++++++ node/node.go | 88 ++++++++++++++++------------------- node/node_handler.go | 2 +- rpc/transaction.go | 2 +- 6 files changed, 81 insertions(+), 58 deletions(-) diff --git a/hmy/blockchain.go b/hmy/blockchain.go index 2ee030986d..45bd099f1f 100644 --- a/hmy/blockchain.go +++ b/hmy/blockchain.go @@ -211,11 +211,21 @@ func (hmy *Harmony) CurrentBlock() *types.Block { return types.NewBlockWithHeader(hmy.BlockChain.CurrentHeader()) } -// GetBlock ... +// CurrentHeader returns the current header from the local chain. +func (hmy *Harmony) CurrentHeader() *block.Header { + return hmy.BlockChain.CurrentHeader() +} + +// GetBlock returns block by hash. func (hmy *Harmony) GetBlock(ctx context.Context, hash common.Hash) (*types.Block, error) { return hmy.BlockChain.GetBlockByHash(hash), nil } +// GetHeader returns header by hash. +func (hmy *Harmony) GetHeader(ctx context.Context, hash common.Hash) (*block.Header, error) { + return hmy.BlockChain.GetHeaderByHash(hash), nil +} + // GetCurrentBadBlocks .. func (hmy *Harmony) GetCurrentBadBlocks() []core.BadBlock { return hmy.BlockChain.BadBlocks() diff --git a/hmy/hmy.go b/hmy/hmy.go index 58e8d59edf..fd01f123c6 100644 --- a/hmy/hmy.go +++ b/hmy/hmy.go @@ -187,16 +187,20 @@ func (hmy *Harmony) ProtocolVersion() int { return proto.ProtocolVersion } -// IsLeader exposes if node is currently leader +// IsLeader exposes if node is currently leader. func (hmy *Harmony) IsLeader() bool { return hmy.NodeAPI.IsCurrentlyLeader() } -// GetNodeMetadata .. +// GetNodeMetadata returns the node metadata. func (hmy *Harmony) GetNodeMetadata() commonRPC.NodeMetadata { - header := hmy.CurrentBlock().Header() - cfg := nodeconfig.GetShardConfig(header.ShardID()) - var blockEpoch *uint64 + var ( + header = hmy.CurrentHeader() + cfg = nodeconfig.GetShardConfig(header.ShardID()) + blockEpoch *uint64 + blsKeys []string + c = commonRPC.C{} + ) if header.ShardID() == shard.BeaconChainShardID { sched := shard.Schedule.InstanceForEpoch(header.Epoch()) @@ -204,13 +208,12 @@ func (hmy *Harmony) GetNodeMetadata() commonRPC.NodeMetadata { blockEpoch = &b } - blsKeys := []string{} if cfg.ConsensusPriKey != nil { for _, key := range cfg.ConsensusPriKey { blsKeys = append(blsKeys, key.Pub.Bytes.Hex()) } } - c := commonRPC.C{} + c.TotalKnownPeers, c.Connected, c.NotConnected = hmy.NodeAPI.PeerConnectivity() syncPeers := hmy.NodeAPI.SyncPeers() diff --git a/internal/registry/registry.go b/internal/registry/registry.go index 025b652a69..98b69a3ef5 100644 --- a/internal/registry/registry.go +++ b/internal/registry/registry.go @@ -10,6 +10,7 @@ import ( type Registry struct { mu sync.Mutex blockchain core.BlockChain + txPool *core.TxPool } // New creates a new registry. @@ -33,3 +34,20 @@ func (r *Registry) GetBlockchain() core.BlockChain { return r.blockchain } + +// SetTxPool sets the txpool to registry. +func (r *Registry) SetTxPool(txPool *core.TxPool) *Registry { + r.mu.Lock() + defer r.mu.Unlock() + + r.txPool = txPool + return r +} + +// GetTxPool gets the txpool from registry. +func (r *Registry) GetTxPool() *core.TxPool { + r.mu.Lock() + defer r.mu.Unlock() + + return r.txPool +} diff --git a/node/node.go b/node/node.go index b41163ac47..c7a01bd76e 100644 --- a/node/node.go +++ b/node/node.go @@ -99,12 +99,11 @@ type ISync interface { // Node represents a protocol-participating node in the network type Node struct { - Consensus *consensus.Consensus // Consensus object containing all Consensus related data (e.g. committee members, signatures, commits) - ConfirmedBlockChannel chan *types.Block // The channel to send confirmed blocks - BeaconBlockChannel chan *types.Block // The channel to send beacon blocks for non-beaconchain nodes - pendingCXReceipts map[string]*types.CXReceiptsProof // All the receipts received but not yet processed for Consensus - pendingCXMutex sync.Mutex - crosslinks *crosslinks.Crosslinks // Memory storage for crosslink processing. + Consensus *consensus.Consensus // Consensus object containing all Consensus related data (e.g. committee members, signatures, commits) + BeaconBlockChannel chan *types.Block // The channel to send beacon blocks for non-beaconchain nodes + pendingCXReceipts map[string]*types.CXReceiptsProof // All the receipts received but not yet processed for Consensus + pendingCXMutex sync.Mutex + crosslinks *crosslinks.Crosslinks // Memory storage for crosslink processing. // Shard databases shardChains shardchain.Collection SelfPeer p2p.Peer @@ -227,8 +226,8 @@ func (node *Node) tryBroadcast(tx *types.Transaction) { utils.Logger().Info().Str("shardGroupID", string(shardGroupID)).Msg("tryBroadcast") for attempt := 0; attempt < NumTryBroadCast; attempt++ { - if err := node.host.SendMessageToGroups([]nodeconfig.GroupID{shardGroupID}, - p2p.ConstructMessage(msg)); err != nil && attempt < NumTryBroadCast { + err := node.host.SendMessageToGroups([]nodeconfig.GroupID{shardGroupID}, p2p.ConstructMessage(msg)) + if err != nil { utils.Logger().Error().Int("attempt", attempt).Msg("Error when trying to broadcast tx") } else { break @@ -246,7 +245,7 @@ func (node *Node) tryBroadcastStaking(stakingTx *staking.StakingTransaction) { for attempt := 0; attempt < NumTryBroadCast; attempt++ { if err := node.host.SendMessageToGroups([]nodeconfig.GroupID{shardGroupID}, - p2p.ConstructMessage(msg)); err != nil && attempt < NumTryBroadCast { + p2p.ConstructMessage(msg)); err != nil { utils.Logger().Error().Int("attempt", attempt).Msg("Error when trying to broadcast staking tx") } else { break @@ -255,31 +254,27 @@ func (node *Node) tryBroadcastStaking(stakingTx *staking.StakingTransaction) { } // Add new transactions to the pending transaction list. -func (node *Node) addPendingTransactions(newTxs types.Transactions) []error { - // if inSync, _, _ := node.SyncStatus(node.Blockchain().ShardID()); !inSync && node.NodeConfig.GetNetworkType() == nodeconfig.Mainnet { - // utils.Logger().Debug(). - // Int("length of newTxs", len(newTxs)). - // Msg("[addPendingTransactions] Node out of sync, ignoring transactions") - // return nil - // } - - poolTxs := types.PoolTransactions{} - errs := []error{} - acceptCx := node.Blockchain().Config().AcceptsCrossTx(node.Blockchain().CurrentHeader().Epoch()) +func addPendingTransactions(registry *registry.Registry, newTxs types.Transactions) []error { + var ( + errs []error + bc = registry.GetBlockchain() + txPool = registry.GetTxPool() + poolTxs = types.PoolTransactions{} + acceptCx = bc.Config().AcceptsCrossTx(bc.CurrentHeader().Epoch()) + ) for _, tx := range newTxs { if tx.ShardID() != tx.ToShardID() && !acceptCx { errs = append(errs, errors.WithMessage(errInvalidEpoch, "cross-shard tx not accepted yet")) continue } - if tx.IsEthCompatible() && !node.Blockchain().Config().IsEthCompatible(node.Blockchain().CurrentBlock().Epoch()) { + if tx.IsEthCompatible() && !bc.Config().IsEthCompatible(bc.CurrentBlock().Epoch()) { errs = append(errs, errors.WithMessage(errInvalidEpoch, "ethereum tx not accepted yet")) continue } poolTxs = append(poolTxs, tx) } - errs = append(errs, node.TxPool.AddRemotes(poolTxs)...) - - pendingCount, queueCount := node.TxPool.Stats() + errs = append(errs, registry.GetTxPool().AddRemotes(poolTxs)...) + pendingCount, queueCount := txPool.Stats() utils.Logger().Debug(). Interface("err", errs). Int("length of newTxs", len(newTxs)). @@ -346,7 +341,7 @@ func (node *Node) AddPendingStakingTransaction( // This is only called from SDK. func (node *Node) AddPendingTransaction(newTx *types.Transaction) error { if newTx.ShardID() == node.NodeConfig.ShardID { - errs := node.addPendingTransactions(types.Transactions{newTx}) + errs := addPendingTransactions(node.registry, types.Transactions{newTx}) var err error for i := range errs { if errs[i] != nil { @@ -544,11 +539,11 @@ func (node *Node) validateNodeMessage(ctx context.Context, payload []byte) ( // validate shardID // validate public key size // verify message signature -func (node *Node) validateShardBoundMessage( - ctx context.Context, payload []byte, +func validateShardBoundMessage(consensus *consensus.Consensus, nodeConfig *nodeconfig.ConfigType, payload []byte, ) (*msg_pb.Message, *bls.SerializedPublicKey, bool, error) { var ( m msg_pb.Message + //consensus = registry.GetConsensus() ) if err := protobuf.Unmarshal(payload, &m); err != nil { nodeConsensusMessageCounterVec.With(prometheus.Labels{"type": "invalid_unmarshal"}).Inc() @@ -556,7 +551,7 @@ func (node *Node) validateShardBoundMessage( } // ignore messages not intended for explorer - if node.NodeConfig.Role() == nodeconfig.ExplorerNode { + if nodeConfig.Role() == nodeconfig.ExplorerNode { switch m.Type { case msg_pb.MessageType_ANNOUNCE, @@ -573,7 +568,7 @@ func (node *Node) validateShardBoundMessage( // in order to avoid possible trap forever but drop PREPARE and COMMIT // which are message types specifically for a node acting as leader // so we just ignore those messages - if node.Consensus.IsViewChangingMode() { + if consensus.IsViewChangingMode() { switch m.Type { case msg_pb.MessageType_PREPARE, msg_pb.MessageType_COMMIT: nodeConsensusMessageCounterVec.With(prometheus.Labels{"type": "ignored"}).Inc() @@ -589,7 +584,7 @@ func (node *Node) validateShardBoundMessage( } // ignore message not intended for leader, but still forward them to the network - if node.Consensus.IsLeader() { + if consensus.IsLeader() { switch m.Type { case msg_pb.MessageType_ANNOUNCE, msg_pb.MessageType_PREPARED, msg_pb.MessageType_COMMITTED: nodeConsensusMessageCounterVec.With(prometheus.Labels{"type": "ignored"}).Inc() @@ -602,7 +597,7 @@ func (node *Node) validateShardBoundMessage( senderBitmap := []byte{} if maybeCon != nil { - if maybeCon.ShardId != node.Consensus.ShardID { + if maybeCon.ShardId != consensus.ShardID { nodeConsensusMessageCounterVec.With(prometheus.Labels{"type": "invalid_shard"}).Inc() return nil, nil, true, errors.WithStack(errWrongShardID) } @@ -612,17 +607,17 @@ func (node *Node) validateShardBoundMessage( senderBitmap = maybeCon.SenderPubkeyBitmap } // If the viewID is too old, reject the message. - if maybeCon.ViewId+5 < node.Consensus.GetCurBlockViewID() { + if maybeCon.ViewId+5 < consensus.GetCurBlockViewID() { return nil, nil, true, errors.WithStack(errViewIDTooOld) } } else if maybeVC != nil { - if maybeVC.ShardId != node.Consensus.ShardID { + if maybeVC.ShardId != consensus.ShardID { nodeConsensusMessageCounterVec.With(prometheus.Labels{"type": "invalid_shard"}).Inc() return nil, nil, true, errors.WithStack(errWrongShardID) } senderKey = maybeVC.SenderPubkey // If the viewID is too old, reject the message. - if maybeVC.ViewId+5 < node.Consensus.GetViewChangingID() { + if maybeVC.ViewId+5 < consensus.GetViewChangingID() { return nil, nil, true, errors.WithStack(errViewIDTooOld) } } else { @@ -632,7 +627,7 @@ func (node *Node) validateShardBoundMessage( // ignore mesage not intended for validator // but still forward them to the network - if !node.Consensus.IsLeader() { + if !consensus.IsLeader() { switch m.Type { case msg_pb.MessageType_PREPARE, msg_pb.MessageType_COMMIT: nodeConsensusMessageCounterVec.With(prometheus.Labels{"type": "ignored"}).Inc() @@ -648,12 +643,12 @@ func (node *Node) validateShardBoundMessage( } copy(serializedKey[:], senderKey) - if !node.Consensus.IsValidatorInCommittee(serializedKey) { + if !consensus.IsValidatorInCommittee(serializedKey) { nodeConsensusMessageCounterVec.With(prometheus.Labels{"type": "invalid_committee"}).Inc() return nil, nil, true, errors.WithStack(shard.ErrValidNotInCommittee) } } else { - count := node.Consensus.Decider.ParticipantsCount() + count := consensus.Decider.ParticipantsCount() if (count+7)>>3 != int64(len(senderBitmap)) { nodeConsensusMessageCounterVec.With(prometheus.Labels{"type": "invalid_participant_count"}).Inc() return nil, nil, true, errors.WithStack(errWrongSizeOfBitmap) @@ -667,7 +662,6 @@ func (node *Node) validateShardBoundMessage( } var ( - errMsgHadNoHMYPayLoadAssumption = errors.New("did not have sufficient size for hmy msg") errConsensusMessageOnUnexpectedTopic = errors.New("received consensus on wrong topic") ) @@ -794,8 +788,8 @@ func (node *Node) StartPubSub() error { nodeP2PMessageCounterVec.With(prometheus.Labels{"type": "consensus_total"}).Inc() // validate consensus message - validMsg, senderPubKey, ignore, err := node.validateShardBoundMessage( - context.TODO(), openBox[proto.MessageCategoryBytes:], + validMsg, senderPubKey, ignore, err := validateShardBoundMessage( + node.Consensus, node.NodeConfig, openBox[proto.MessageCategoryBytes:], ) if err != nil { @@ -1031,13 +1025,11 @@ func New( crosslinks: crosslinks.New(), syncID: GenerateSyncID(), } - - // Get the node config that's created in the harmony.go program. - if consensusObj != nil { - node.NodeConfig = nodeconfig.GetShardConfig(consensusObj.ShardID) - } else { - node.NodeConfig = nodeconfig.GetDefaultConfig() + if consensusObj == nil { + panic("consensusObj is nil") } + // Get the node config that's created in the harmony.go program. + node.NodeConfig = nodeconfig.GetShardConfig(consensusObj.ShardID) node.HarmonyConfig = harmonyconfig if host != nil { @@ -1051,7 +1043,7 @@ func New( node.shardChains = collection node.IsSynchronized = abool.NewBool(false) - if host != nil && consensusObj != nil { + if host != nil { // Consensus and associated channel to communicate blocks node.Consensus = consensusObj @@ -1080,7 +1072,6 @@ func New( } } - node.ConfirmedBlockChannel = make(chan *types.Block) node.BeaconBlockChannel = make(chan *types.Block) txPoolConfig := core.DefaultTxPoolConfig @@ -1114,6 +1105,7 @@ func New( } node.TxPool = core.NewTxPool(txPoolConfig, node.Blockchain().Config(), blockchain, node.TransactionErrorSink) + node.registry.SetTxPool(node.TxPool) node.CxPool = core.NewCxPool(core.CxPoolSize) node.Worker = worker.New(node.Blockchain().Config(), blockchain, beaconChain, engine) diff --git a/node/node_handler.go b/node/node_handler.go index 3589d19660..acc4afc266 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -116,7 +116,7 @@ func (node *Node) transactionMessageHandler(msgPayload []byte) { Msg("Failed to deserialize transaction list") return } - node.addPendingTransactions(txs) + addPendingTransactions(node.registry, txs) } } diff --git a/rpc/transaction.go b/rpc/transaction.go index 92e9746954..8ea211d6a9 100644 --- a/rpc/transaction.go +++ b/rpc/transaction.go @@ -203,7 +203,7 @@ func (s *PublicTransactionService) GetTransactionByHash( DoMetricRPCQueryInfo(GetTransactionByHash, FailedNumber) return nil, nil } - block, err := s.hmy.GetBlock(ctx, blockHash) + block, err := s.hmy.GetHeader(ctx, blockHash) if err != nil { utils.Logger().Debug(). Err(err). From 3ea6ebfa7dc183bc99064262de1fd5c492458b41 Mon Sep 17 00:00:00 2001 From: Gheis Mohammadi Date: Wed, 5 Apr 2023 10:46:22 +0800 Subject: [PATCH 388/420] add prefix for contract code (#4397) --- core/rawdb/schema.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 2afc496122..34e818b32e 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -183,8 +183,7 @@ func skeletonHeaderKey(number uint64) []byte { // codeKey = CodePrefix + hash func codeKey(hash common.Hash) []byte { - // We don't use any prefix for code key, otherwise we should return append(CodePrefix, hash.Bytes()...) - return hash.Bytes() + return append(CodePrefix, hash.Bytes()...) } // IsCodeKey reports whether the given byte slice is the key of contract code, From 7d47d9e8a9e728399d75199b966aaadf52a4e57a Mon Sep 17 00:00:00 2001 From: Konstantin <355847+Frozen@users.noreply.github.com> Date: Wed, 5 Apr 2023 19:51:05 -0300 Subject: [PATCH 389/420] Rotate external validators for non-beacon shards. (#4373) * Rotate only non beacon shards. * Rotate all shards, but only hmy validators for beacon. * Fix type. * Revert "Fix type." This reverts commit 0a8b506c763d9f8609abff7395ba32b18e43b149. * Revert "Rotate all shards, but only hmy validators for beacon." This reverts commit 70b09e2de81aa2cbffae3ccdfd4e334e7d938759. * Fixed failed test. * Revert "Revert "Rotate all shards, but only hmy validators for beacon."" This reverts commit 66cfaa9817488be60ed5b5cfee1fe833ede237c8. * Frequency by slots count. * Fix config. * First validator produce rest blocks. * Updated. * Add lock. --- consensus/consensus.go | 2 + consensus/consensus_service.go | 2 +- consensus/consensus_v2.go | 85 ++-- consensus/validator.go | 3 +- consensus/view_change.go | 8 +- core/blockchain.go | 2 + core/blockchain_impl.go | 56 ++- core/blockchain_stub.go | 4 + core/rawdb/accessors_metadata.go | 34 ++ core/rawdb/accessors_metadata_test.go | 32 ++ core/rawdb/schema.go | 5 + internal/configs/sharding/shardingconfig.go | 3 +- internal/params/config.go | 468 ++++++++++---------- numeric/decimal.go | 6 + numeric/decimal_test.go | 24 + 15 files changed, 468 insertions(+), 266 deletions(-) create mode 100644 core/rawdb/accessors_metadata_test.go diff --git a/consensus/consensus.go b/consensus/consensus.go index e0cc591dfe..d5130e22c9 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -218,6 +218,8 @@ func (consensus *Consensus) getConsensusLeaderPrivateKey() (*bls.PrivateKeyWrapp // SetBlockVerifier sets the block verifier func (consensus *Consensus) SetBlockVerifier(verifier VerifyBlockFunc) { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() consensus.BlockVerifier = verifier consensus.vc.SetVerifyBlock(consensus.verifyBlock) } diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 0e4bb68141..3ca29812d9 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -547,7 +547,7 @@ func (consensus *Consensus) GetFinality() int64 { return consensus.finality } -// switchPhase will switch FBFTPhase to nextPhase if the desirePhase equals the nextPhase +// switchPhase will switch FBFTPhase to desired phase. func (consensus *Consensus) switchPhase(subject string, desired FBFTPhase) { consensus.getLogger().Info(). Str("from:", consensus.phase.String()). diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 99bccf7554..b00e0d0d30 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -13,7 +13,6 @@ import ( "github.com/harmony-one/harmony/consensus/signature" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" - "github.com/rs/zerolog" msg_pb "github.com/harmony-one/harmony/api/proto/message" @@ -688,40 +687,70 @@ func (consensus *Consensus) commitBlock(blk *types.Block, committedMsg *FBFTMess // rotateLeader rotates the leader to the next leader in the committee. // This function must be called with enabled leader rotation. func (consensus *Consensus) rotateLeader(epoch *big.Int) { - prev := consensus.getLeaderPubKey() - bc := consensus.Blockchain() - curNumber := bc.CurrentHeader().Number().Uint64() - utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v numblocks:%d", epoch.Uint64(), bc.Config().IsLeaderRotation(epoch), bc.Config().LeaderRotationBlocksCount) - leader := consensus.getLeaderPubKey() - for i := 0; i < bc.Config().LeaderRotationBlocksCount; i++ { - header := bc.GetHeaderByNumber(curNumber - uint64(i)) - if header == nil { - return - } - // Previous epoch, we should not change leader. - if header.Epoch().Uint64() != epoch.Uint64() { - return - } - // Check if the same leader. - pub, err := bc.GetLeaderPubKeyFromCoinbase(header) - if err != nil { - utils.Logger().Error().Err(err).Msg("Failed to get leader public key from coinbase") - return - } - if !pub.Object.IsEqual(leader.Object) { - // Another leader. - return - } + var ( + bc = consensus.Blockchain() + prev = consensus.getLeaderPubKey() + leader = consensus.getLeaderPubKey() + ) + utils.Logger().Info().Msgf("[Rotating leader] epoch: %v rotation:%v external rotation %v", epoch.Uint64(), bc.Config().IsLeaderRotation(epoch), bc.Config().IsLeaderRotationExternalValidatorsAllowed(epoch, consensus.ShardID)) + ss, err := bc.ReadShardState(epoch) + if err != nil { + utils.Logger().Error().Err(err).Msg("Failed to read shard state") + return + } + committee, err := ss.FindCommitteeByID(consensus.ShardID) + if err != nil { + utils.Logger().Error().Err(err).Msg("Failed to find committee") + return + } + slotsCount := len(committee.Slots) + blocksPerEpoch := shard.Schedule.InstanceForEpoch(epoch).BlocksPerEpoch() + if blocksPerEpoch == 0 { + utils.Logger().Error().Msg("[Rotating leader] blocks per epoch is 0") + return + } + if slotsCount == 0 { + utils.Logger().Error().Msg("[Rotating leader] slots count is 0") + return + } + numBlocksProducedByLeader := blocksPerEpoch / uint64(slotsCount) + rest := blocksPerEpoch % uint64(slotsCount) + const minimumBlocksForLeaderInRow = 3 + if numBlocksProducedByLeader < minimumBlocksForLeaderInRow { + // mine no less than 3 blocks in a row + numBlocksProducedByLeader = minimumBlocksForLeaderInRow + } + type stored struct { + pub []byte + epoch uint64 + count uint64 + shifts uint64 // count how much changes validator per epoch + } + var s stored + s.pub, s.epoch, s.count, s.shifts, _ = bc.LeaderRotationMeta() + if !bytes.Equal(leader.Bytes[:], s.pub) { + // Another leader. + return + } + // if it is the first validator which produce blocks, then it should produce `rest` blocks too. + if s.shifts == 0 { + numBlocksProducedByLeader += rest + } + if s.count < numBlocksProducedByLeader { + // Not enough blocks produced by the leader. + return } // Passed all checks, we can change leader. + // NthNext will move the leader to the next leader in the committee. + // It does not know anything about external or internal validators. var ( wasFound bool next *bls.PublicKeyWrapper ) - if consensus.ShardID == shard.BeaconChainShardID { - wasFound, next = consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) - } else { + if bc.Config().IsLeaderRotationExternalValidatorsAllowed(epoch, consensus.ShardID) { wasFound, next = consensus.Decider.NthNext(leader, 1) + } else { + wasFound, next = consensus.Decider.NthNextHmy(shard.Schedule.InstanceForEpoch(epoch), leader, 1) } if !wasFound { utils.Logger().Error().Msg("Failed to get next leader") diff --git a/consensus/validator.go b/consensus/validator.go index f85cb8e3d8..92008a91e6 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -63,9 +63,8 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { go func() { // Best effort check, no need to error out. _, err := consensus.ValidateNewBlock(recvMsg) - if err == nil { - consensus.getLogger().Info(). + consensus.GetLogger().Info(). Msg("[Announce] Block verified") } }() diff --git a/consensus/view_change.go b/consensus/view_change.go index aafdfd1210..45838f9bc4 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -203,13 +203,13 @@ func (consensus *Consensus) getNextLeaderKey(viewID uint64) *bls.PublicKeyWrappe var wasFound bool var next *bls.PublicKeyWrapper if blockchain != nil && blockchain.Config().IsLeaderRotation(epoch) { - if consensus.ShardID == shard.BeaconChainShardID { - wasFound, next = consensus.Decider.NthNextHmy( - shard.Schedule.InstanceForEpoch(epoch), + if blockchain.Config().IsLeaderRotationExternalValidatorsAllowed(epoch, consensus.ShardID) { + wasFound, next = consensus.Decider.NthNext( lastLeaderPubKey, gap) } else { - wasFound, next = consensus.Decider.NthNext( + wasFound, next = consensus.Decider.NthNextHmy( + shard.Schedule.InstanceForEpoch(epoch), lastLeaderPubKey, gap) } diff --git a/core/blockchain.go b/core/blockchain.go index fda4831655..8afe622c70 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -107,6 +107,8 @@ type BlockChain interface { // // After insertion is done, all accumulated events will be fired. InsertChain(chain types.Blocks, verifyHeaders bool) (int, error) + // LeaderRotationMeta returns the number of continuous blocks by the leader. + LeaderRotationMeta() (publicKeyBytes []byte, epoch, count, shifts uint64, err error) // BadBlocks returns a list of the last 'bad blocks' that // the client has seen on the network. BadBlocks() []BadBlock diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 148176f236..f1d0968422 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -114,6 +114,7 @@ const ( validatorListByDelegatorCacheLimit = 128 pendingCrossLinksCacheLimit = 2 blockAccumulatorCacheLimit = 64 + leaderPubKeyFromCoinbaseLimit = 8 maxPendingSlashes = 256 // BlockChainVersion ensures that an incompatible database forces a resync from scratch. BlockChainVersion = 3 @@ -240,7 +241,7 @@ func newBlockChainWithOptions( validatorListByDelegatorCache, _ := lru.New(validatorListByDelegatorCacheLimit) pendingCrossLinksCache, _ := lru.New(pendingCrossLinksCacheLimit) blockAccumulatorCache, _ := lru.New(blockAccumulatorCacheLimit) - leaderPubKeyFromCoinbase, _ := lru.New(chainConfig.LeaderRotationBlocksCount + 2) + leaderPubKeyFromCoinbase, _ := lru.New(leaderPubKeyFromCoinbaseLimit) bc := &BlockChainImpl{ chainConfig: chainConfig, @@ -1522,6 +1523,18 @@ func (bc *BlockChainImpl) InsertChain(chain types.Blocks, verifyHeaders bool) (i n, events, logs, err := bc.insertChain(chain, verifyHeaders) bc.PostChainEvents(events, logs) + if err == nil { + // there should be only 1 block. + for _, b := range chain { + if b.Epoch().Uint64() > 0 { + err := bc.saveLeaderRotationMeta(b.Header()) + if err != nil { + utils.Logger().Error().Err(err).Msg("save leader continuous blocks count error") + return n, err + } + } + } + } if bc.isInitTiKV() && err != nil { // if has some error, master writer node will release the permission _, _ = bc.redisPreempt.Unlock() @@ -1529,6 +1542,47 @@ func (bc *BlockChainImpl) InsertChain(chain types.Blocks, verifyHeaders bool) (i return n, err } +func (bc *BlockChainImpl) saveLeaderRotationMeta(h *block.Header) error { + blockPubKey, err := bc.getLeaderPubKeyFromCoinbase(h) + if err != nil { + return err + } + type stored struct { + pub []byte + epoch uint64 + count uint64 + shifts uint64 + } + var s stored + // error is possible here only on the first iteration, so we can ignore it + s.pub, s.epoch, s.count, s.shifts, _ = rawdb.ReadLeaderRotationMeta(bc.db) + + // increase counter only if the same leader and epoch + if bytes.Equal(s.pub, blockPubKey.Bytes[:]) && s.epoch == h.Epoch().Uint64() { + s.count++ + } else { + s.count = 1 + } + // we should increase shifts if the leader is changed. + if !bytes.Equal(s.pub, blockPubKey.Bytes[:]) { + s.shifts++ + } + // but set to zero if new + if s.epoch != h.Epoch().Uint64() { + s.shifts = 0 + } + + err = rawdb.WriteLeaderRotationMeta(bc.db, blockPubKey.Bytes[:], h.Epoch().Uint64(), s.count, s.shifts) + if err != nil { + return err + } + return nil +} + +func (bc *BlockChainImpl) LeaderRotationMeta() (publicKeyBytes []byte, epoch, count, shifts uint64, err error) { + return rawdb.ReadLeaderRotationMeta(bc.db) +} + // insertChain will execute the actual chain insertion and event aggregation. The // only reason this method exists as a separate one is to make locking cleaner // with deferred statements. diff --git a/core/blockchain_stub.go b/core/blockchain_stub.go index f9e9111eae..cfe72eed6d 100644 --- a/core/blockchain_stub.go +++ b/core/blockchain_stub.go @@ -423,3 +423,7 @@ func (a Stub) SyncFromTiKVWriter(newBlkNum uint64, logs []*types.Log) error { func (a Stub) InitTiKV(conf *harmonyconfig.TiKVConfig) { return } + +func (a Stub) LeaderRotationMeta() (publicKeyBytes []byte, epoch, count, shifts uint64, err error) { + return nil, 0, 0, 0, errors.Errorf("method LeaderRotationMeta not implemented for %s", a.Name) +} diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index 6fef26d09e..69a7522f6c 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -17,14 +17,17 @@ package rawdb import ( + "encoding/binary" "encoding/json" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" + "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/utils" + "github.com/pkg/errors" ) // ReadDatabaseVersion retrieves the version number of the database. @@ -192,3 +195,34 @@ func WriteTransitionStatus(db ethdb.KeyValueWriter, data []byte) { utils.Logger().Error().Err(err).Msg("Failed to store the eth2 transition status") } } + +// WriteLeaderRotationMeta writes the leader continuous blocks count to the database. +func WriteLeaderRotationMeta(db DatabaseWriter, leader []byte, epoch uint64, count, shifts uint64) error { + if len(leader) != bls.PublicKeySizeInBytes { + return errors.New("invalid leader public key size") + } + value := make([]byte, bls.PublicKeySizeInBytes+8*3) + copy(value, leader) + binary.LittleEndian.PutUint64(value[len(leader)+8*0:], epoch) + binary.LittleEndian.PutUint64(value[len(leader)+8*1:], count) + binary.LittleEndian.PutUint64(value[len(leader)+8*2:], shifts) + if err := db.Put(leaderContinuousBlocksCountKey(), value); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store leader continuous blocks count") + return err + } + return nil +} + +// ReadLeaderRotationMeta retrieves the leader continuous blocks count from the database. +func ReadLeaderRotationMeta(db DatabaseReader) (pubKeyBytes []byte, epoch, count, shifts uint64, err error) { + data, _ := db.Get(leaderContinuousBlocksCountKey()) + if len(data) != bls.PublicKeySizeInBytes+24 { + return nil, 0, 0, 0, errors.New("invalid leader continuous blocks count") + } + + pubKeyBytes = data[:bls.PublicKeySizeInBytes] + epoch = binary.LittleEndian.Uint64(data[bls.PublicKeySizeInBytes:]) + count = binary.LittleEndian.Uint64(data[bls.PublicKeySizeInBytes+8:]) + shifts = binary.LittleEndian.Uint64(data[bls.PublicKeySizeInBytes+16:]) + return pubKeyBytes, epoch, count, shifts, nil +} diff --git a/core/rawdb/accessors_metadata_test.go b/core/rawdb/accessors_metadata_test.go new file mode 100644 index 0000000000..62cfac062c --- /dev/null +++ b/core/rawdb/accessors_metadata_test.go @@ -0,0 +1,32 @@ +package rawdb + +import ( + "testing" + + ethRawDB "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/harmony-one/harmony/crypto/bls" +) + +func TestLeaderRotationMeta(t *testing.T) { + db := ethRawDB.NewMemoryDatabase() + err := WriteLeaderRotationMeta(db, make([]byte, bls.PublicKeySizeInBytes), 1, 2, 3) + if err != nil { + t.Fatal(err) + } + pub, epoch, count, shifts, err := ReadLeaderRotationMeta(db) + if err != nil { + t.Fatal(err) + } + if len(pub) != bls.PublicKeySizeInBytes { + t.Fatal("invalid leader public key size") + } + if epoch != 1 { + t.Fatal("invalid epoch") + } + if count != 2 { + t.Fatal("invalid count") + } + if shifts != 3 { + t.Fatal("invalid shifts") + } +} diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 34e818b32e..798f27f59e 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -100,6 +100,7 @@ var ( pendingCrosslinkKey = []byte("pendingCL") // prefix for shard last pending crosslink pendingSlashingKey = []byte("pendingSC") // prefix for shard last pending slashing record preimagePrefix = []byte("secure-key-") // preimagePrefix + hash -> preimage + continuousBlocksCountKey = []byte("continuous") // key for continuous blocks count configPrefix = []byte("ethereum-config-") // config prefix for the db crosslinkPrefix = []byte("cl") // prefix for crosslink delegatorValidatorListPrefix = []byte("dvl") // prefix for delegator's validator list @@ -285,6 +286,10 @@ func preimageKey(hash common.Hash) []byte { return append(preimagePrefix, hash.Bytes()...) } +func leaderContinuousBlocksCountKey() []byte { + return continuousBlocksCountKey +} + // configKey = configPrefix + hash func configKey(hash common.Hash) []byte { return append(configPrefix, hash.Bytes()...) diff --git a/internal/configs/sharding/shardingconfig.go b/internal/configs/sharding/shardingconfig.go index 30b4dca4c3..91b4458dd3 100644 --- a/internal/configs/sharding/shardingconfig.go +++ b/internal/configs/sharding/shardingconfig.go @@ -72,8 +72,9 @@ type Instance interface { // ReshardingEpoch returns a list of Epoch while off-chain resharding happens ReshardingEpoch() []*big.Int - // Count of blocks per epoch + // BlocksPerEpoch returns the number of blocks per epoch. BlocksPerEpoch() uint64 + // HIP-16: The absolute number of maximum effective slots per shard limit for each validator. 0 means no limit. SlotsLimit() int diff --git a/internal/params/config.go b/internal/params/config.go index d900386303..a34dcd3973 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -36,250 +36,250 @@ var once sync.Once var ( // MainnetChainConfig is the chain parameters to run a node on the main network. MainnetChainConfig = &ChainConfig{ - ChainID: MainnetChainID, - EthCompatibleChainID: EthMainnetShard0ChainID, - EthCompatibleShard0ChainID: EthMainnetShard0ChainID, - EthCompatibleEpoch: big.NewInt(442), // Around Thursday Feb 4th 2020, 10AM PST - CrossTxEpoch: big.NewInt(28), - CrossLinkEpoch: big.NewInt(186), - AggregatedRewardEpoch: big.NewInt(689), // Around Wed Sept 15th 2021 with 3.5s block time - StakingEpoch: big.NewInt(186), - PreStakingEpoch: big.NewInt(185), - QuickUnlockEpoch: big.NewInt(191), - FiveSecondsEpoch: big.NewInt(230), - TwoSecondsEpoch: big.NewInt(366), // Around Tuesday Dec 8th 2020, 8AM PST - SixtyPercentEpoch: big.NewInt(530), // Around Monday Apr 12th 2021, 22:30 UTC - RedelegationEpoch: big.NewInt(290), - NoEarlyUnlockEpoch: big.NewInt(530), // Around Monday Apr 12th 2021, 22:30 UTC - VRFEpoch: big.NewInt(631), // Around Wed July 7th 2021 - PrevVRFEpoch: big.NewInt(689), // Around Wed Sept 15th 2021 with 3.5s block time - MinDelegation100Epoch: big.NewInt(631), // Around Wed July 7th 2021 - MinCommissionRateEpoch: big.NewInt(631), // Around Wed July 7th 2021 - MinCommissionPromoPeriod: big.NewInt(100), - EPoSBound35Epoch: big.NewInt(631), // Around Wed July 7th 2021 - EIP155Epoch: big.NewInt(28), - S3Epoch: big.NewInt(28), - DataCopyFixEpoch: big.NewInt(689), // Around Wed Sept 15th 2021 with 3.5s block time - IstanbulEpoch: big.NewInt(314), - ReceiptLogEpoch: big.NewInt(101), - SHA3Epoch: big.NewInt(725), // Around Mon Oct 11 2021, 19:00 UTC - HIP6And8Epoch: big.NewInt(725), // Around Mon Oct 11 2021, 19:00 UTC - StakingPrecompileEpoch: big.NewInt(871), // Around Tue Feb 11 2022 - ChainIdFixEpoch: big.NewInt(1323), // Around Wed 8 Feb 11:30PM UTC - SlotsLimitedEpoch: big.NewInt(999), // Around Fri, 27 May 2022 09:41:02 UTC with 2s block time - CrossShardXferPrecompileEpoch: big.NewInt(1323), // Around Wed 8 Feb 11:30PM UTC - AllowlistEpoch: EpochTBD, - FeeCollectEpoch: EpochTBD, - LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 64, - ValidatorCodeFixEpoch: EpochTBD, + ChainID: MainnetChainID, + EthCompatibleChainID: EthMainnetShard0ChainID, + EthCompatibleShard0ChainID: EthMainnetShard0ChainID, + EthCompatibleEpoch: big.NewInt(442), // Around Thursday Feb 4th 2020, 10AM PST + CrossTxEpoch: big.NewInt(28), + CrossLinkEpoch: big.NewInt(186), + AggregatedRewardEpoch: big.NewInt(689), // Around Wed Sept 15th 2021 with 3.5s block time + StakingEpoch: big.NewInt(186), + PreStakingEpoch: big.NewInt(185), + QuickUnlockEpoch: big.NewInt(191), + FiveSecondsEpoch: big.NewInt(230), + TwoSecondsEpoch: big.NewInt(366), // Around Tuesday Dec 8th 2020, 8AM PST + SixtyPercentEpoch: big.NewInt(530), // Around Monday Apr 12th 2021, 22:30 UTC + RedelegationEpoch: big.NewInt(290), + NoEarlyUnlockEpoch: big.NewInt(530), // Around Monday Apr 12th 2021, 22:30 UTC + VRFEpoch: big.NewInt(631), // Around Wed July 7th 2021 + PrevVRFEpoch: big.NewInt(689), // Around Wed Sept 15th 2021 with 3.5s block time + MinDelegation100Epoch: big.NewInt(631), // Around Wed July 7th 2021 + MinCommissionRateEpoch: big.NewInt(631), // Around Wed July 7th 2021 + MinCommissionPromoPeriod: big.NewInt(100), + EPoSBound35Epoch: big.NewInt(631), // Around Wed July 7th 2021 + EIP155Epoch: big.NewInt(28), + S3Epoch: big.NewInt(28), + DataCopyFixEpoch: big.NewInt(689), // Around Wed Sept 15th 2021 with 3.5s block time + IstanbulEpoch: big.NewInt(314), + ReceiptLogEpoch: big.NewInt(101), + SHA3Epoch: big.NewInt(725), // Around Mon Oct 11 2021, 19:00 UTC + HIP6And8Epoch: big.NewInt(725), // Around Mon Oct 11 2021, 19:00 UTC + StakingPrecompileEpoch: big.NewInt(871), // Around Tue Feb 11 2022 + ChainIdFixEpoch: big.NewInt(1323), // Around Wed 8 Feb 11:30PM UTC + SlotsLimitedEpoch: big.NewInt(999), // Around Fri, 27 May 2022 09:41:02 UTC with 2s block time + CrossShardXferPrecompileEpoch: big.NewInt(1323), // Around Wed 8 Feb 11:30PM UTC + AllowlistEpoch: EpochTBD, + FeeCollectEpoch: EpochTBD, + LeaderRotationExternalNonBeaconLeaders: EpochTBD, + LeaderRotationExternalBeaconLeaders: EpochTBD, + ValidatorCodeFixEpoch: EpochTBD, } // TestnetChainConfig contains the chain parameters to run a node on the harmony test network. TestnetChainConfig = &ChainConfig{ - ChainID: TestnetChainID, - EthCompatibleChainID: EthTestnetShard0ChainID, - EthCompatibleShard0ChainID: EthTestnetShard0ChainID, - EthCompatibleEpoch: big.NewInt(0), - CrossTxEpoch: big.NewInt(0), - CrossLinkEpoch: big.NewInt(2), - AggregatedRewardEpoch: big.NewInt(2), - StakingEpoch: big.NewInt(2), - PreStakingEpoch: big.NewInt(1), - QuickUnlockEpoch: big.NewInt(0), - FiveSecondsEpoch: big.NewInt(0), - TwoSecondsEpoch: big.NewInt(2), - SixtyPercentEpoch: big.NewInt(2), - RedelegationEpoch: big.NewInt(2), - NoEarlyUnlockEpoch: big.NewInt(2), - VRFEpoch: big.NewInt(2), - PrevVRFEpoch: big.NewInt(2), - MinDelegation100Epoch: big.NewInt(2), - MinCommissionRateEpoch: big.NewInt(2), - MinCommissionPromoPeriod: big.NewInt(2), - EPoSBound35Epoch: big.NewInt(2), - EIP155Epoch: big.NewInt(0), - S3Epoch: big.NewInt(0), - DataCopyFixEpoch: big.NewInt(0), - IstanbulEpoch: big.NewInt(0), - ReceiptLogEpoch: big.NewInt(0), - SHA3Epoch: big.NewInt(0), - HIP6And8Epoch: big.NewInt(2), - StakingPrecompileEpoch: big.NewInt(2), - SlotsLimitedEpoch: big.NewInt(2), - ChainIdFixEpoch: big.NewInt(0), - CrossShardXferPrecompileEpoch: big.NewInt(2), - AllowlistEpoch: big.NewInt(2), - LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 64, - FeeCollectEpoch: EpochTBD, - ValidatorCodeFixEpoch: EpochTBD, + ChainID: TestnetChainID, + EthCompatibleChainID: EthTestnetShard0ChainID, + EthCompatibleShard0ChainID: EthTestnetShard0ChainID, + EthCompatibleEpoch: big.NewInt(0), + CrossTxEpoch: big.NewInt(0), + CrossLinkEpoch: big.NewInt(2), + AggregatedRewardEpoch: big.NewInt(2), + StakingEpoch: big.NewInt(2), + PreStakingEpoch: big.NewInt(1), + QuickUnlockEpoch: big.NewInt(0), + FiveSecondsEpoch: big.NewInt(0), + TwoSecondsEpoch: big.NewInt(2), + SixtyPercentEpoch: big.NewInt(2), + RedelegationEpoch: big.NewInt(2), + NoEarlyUnlockEpoch: big.NewInt(2), + VRFEpoch: big.NewInt(2), + PrevVRFEpoch: big.NewInt(2), + MinDelegation100Epoch: big.NewInt(2), + MinCommissionRateEpoch: big.NewInt(2), + MinCommissionPromoPeriod: big.NewInt(2), + EPoSBound35Epoch: big.NewInt(2), + EIP155Epoch: big.NewInt(0), + S3Epoch: big.NewInt(0), + DataCopyFixEpoch: big.NewInt(0), + IstanbulEpoch: big.NewInt(0), + ReceiptLogEpoch: big.NewInt(0), + SHA3Epoch: big.NewInt(0), + HIP6And8Epoch: big.NewInt(2), + StakingPrecompileEpoch: big.NewInt(2), + SlotsLimitedEpoch: big.NewInt(2), + ChainIdFixEpoch: big.NewInt(0), + CrossShardXferPrecompileEpoch: big.NewInt(2), + AllowlistEpoch: big.NewInt(2), + LeaderRotationExternalNonBeaconLeaders: EpochTBD, + LeaderRotationExternalBeaconLeaders: EpochTBD, + FeeCollectEpoch: EpochTBD, + ValidatorCodeFixEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. // All features except for CrossLink are enabled at launch. PangaeaChainConfig = &ChainConfig{ - ChainID: PangaeaChainID, - EthCompatibleChainID: EthPangaeaShard0ChainID, - EthCompatibleShard0ChainID: EthPangaeaShard0ChainID, - EthCompatibleEpoch: big.NewInt(0), - CrossTxEpoch: big.NewInt(0), - CrossLinkEpoch: big.NewInt(2), - AggregatedRewardEpoch: big.NewInt(3), - StakingEpoch: big.NewInt(2), - PreStakingEpoch: big.NewInt(1), - QuickUnlockEpoch: big.NewInt(0), - FiveSecondsEpoch: big.NewInt(0), - TwoSecondsEpoch: big.NewInt(0), - SixtyPercentEpoch: big.NewInt(0), - RedelegationEpoch: big.NewInt(0), - NoEarlyUnlockEpoch: big.NewInt(0), - VRFEpoch: big.NewInt(0), - PrevVRFEpoch: big.NewInt(0), - MinDelegation100Epoch: big.NewInt(0), - MinCommissionRateEpoch: big.NewInt(0), - MinCommissionPromoPeriod: big.NewInt(10), - EPoSBound35Epoch: big.NewInt(0), - EIP155Epoch: big.NewInt(0), - S3Epoch: big.NewInt(0), - DataCopyFixEpoch: big.NewInt(0), - IstanbulEpoch: big.NewInt(0), - ReceiptLogEpoch: big.NewInt(0), - SHA3Epoch: big.NewInt(0), - HIP6And8Epoch: big.NewInt(0), - StakingPrecompileEpoch: big.NewInt(2), // same as staking - ChainIdFixEpoch: big.NewInt(0), - SlotsLimitedEpoch: EpochTBD, // epoch to enable HIP-16 - CrossShardXferPrecompileEpoch: big.NewInt(1), - AllowlistEpoch: EpochTBD, - LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 64, - FeeCollectEpoch: EpochTBD, - ValidatorCodeFixEpoch: EpochTBD, + ChainID: PangaeaChainID, + EthCompatibleChainID: EthPangaeaShard0ChainID, + EthCompatibleShard0ChainID: EthPangaeaShard0ChainID, + EthCompatibleEpoch: big.NewInt(0), + CrossTxEpoch: big.NewInt(0), + CrossLinkEpoch: big.NewInt(2), + AggregatedRewardEpoch: big.NewInt(3), + StakingEpoch: big.NewInt(2), + PreStakingEpoch: big.NewInt(1), + QuickUnlockEpoch: big.NewInt(0), + FiveSecondsEpoch: big.NewInt(0), + TwoSecondsEpoch: big.NewInt(0), + SixtyPercentEpoch: big.NewInt(0), + RedelegationEpoch: big.NewInt(0), + NoEarlyUnlockEpoch: big.NewInt(0), + VRFEpoch: big.NewInt(0), + PrevVRFEpoch: big.NewInt(0), + MinDelegation100Epoch: big.NewInt(0), + MinCommissionRateEpoch: big.NewInt(0), + MinCommissionPromoPeriod: big.NewInt(10), + EPoSBound35Epoch: big.NewInt(0), + EIP155Epoch: big.NewInt(0), + S3Epoch: big.NewInt(0), + DataCopyFixEpoch: big.NewInt(0), + IstanbulEpoch: big.NewInt(0), + ReceiptLogEpoch: big.NewInt(0), + SHA3Epoch: big.NewInt(0), + HIP6And8Epoch: big.NewInt(0), + StakingPrecompileEpoch: big.NewInt(2), // same as staking + ChainIdFixEpoch: big.NewInt(0), + SlotsLimitedEpoch: EpochTBD, // epoch to enable HIP-16 + CrossShardXferPrecompileEpoch: big.NewInt(1), + AllowlistEpoch: EpochTBD, + LeaderRotationExternalNonBeaconLeaders: EpochTBD, + LeaderRotationExternalBeaconLeaders: EpochTBD, + FeeCollectEpoch: EpochTBD, + ValidatorCodeFixEpoch: EpochTBD, } // PartnerChainConfig contains the chain parameters for the Partner network. // This is the Devnet config PartnerChainConfig = &ChainConfig{ - ChainID: PartnerChainID, - EthCompatibleChainID: EthPartnerShard0ChainID, - EthCompatibleShard0ChainID: EthPartnerShard0ChainID, - EthCompatibleEpoch: big.NewInt(0), - CrossTxEpoch: big.NewInt(0), - CrossLinkEpoch: big.NewInt(2), - AggregatedRewardEpoch: big.NewInt(3), - StakingEpoch: big.NewInt(2), - PreStakingEpoch: big.NewInt(1), - QuickUnlockEpoch: big.NewInt(0), - FiveSecondsEpoch: big.NewInt(0), - TwoSecondsEpoch: big.NewInt(0), - SixtyPercentEpoch: big.NewInt(4), - RedelegationEpoch: big.NewInt(0), - NoEarlyUnlockEpoch: big.NewInt(0), - VRFEpoch: big.NewInt(0), - PrevVRFEpoch: big.NewInt(0), - MinDelegation100Epoch: big.NewInt(0), - MinCommissionRateEpoch: big.NewInt(0), - MinCommissionPromoPeriod: big.NewInt(10), - EPoSBound35Epoch: big.NewInt(0), - EIP155Epoch: big.NewInt(0), - S3Epoch: big.NewInt(0), - DataCopyFixEpoch: big.NewInt(0), - IstanbulEpoch: big.NewInt(0), - ReceiptLogEpoch: big.NewInt(0), - SHA3Epoch: big.NewInt(0), - HIP6And8Epoch: big.NewInt(0), - StakingPrecompileEpoch: big.NewInt(2), - ChainIdFixEpoch: big.NewInt(0), - SlotsLimitedEpoch: EpochTBD, // epoch to enable HIP-16 - CrossShardXferPrecompileEpoch: big.NewInt(1), - AllowlistEpoch: EpochTBD, - FeeCollectEpoch: big.NewInt(574), - LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 64, - ValidatorCodeFixEpoch: EpochTBD, + ChainID: PartnerChainID, + EthCompatibleChainID: EthPartnerShard0ChainID, + EthCompatibleShard0ChainID: EthPartnerShard0ChainID, + EthCompatibleEpoch: big.NewInt(0), + CrossTxEpoch: big.NewInt(0), + CrossLinkEpoch: big.NewInt(2), + AggregatedRewardEpoch: big.NewInt(3), + StakingEpoch: big.NewInt(2), + PreStakingEpoch: big.NewInt(1), + QuickUnlockEpoch: big.NewInt(0), + FiveSecondsEpoch: big.NewInt(0), + TwoSecondsEpoch: big.NewInt(0), + SixtyPercentEpoch: big.NewInt(4), + RedelegationEpoch: big.NewInt(0), + NoEarlyUnlockEpoch: big.NewInt(0), + VRFEpoch: big.NewInt(0), + PrevVRFEpoch: big.NewInt(0), + MinDelegation100Epoch: big.NewInt(0), + MinCommissionRateEpoch: big.NewInt(0), + MinCommissionPromoPeriod: big.NewInt(10), + EPoSBound35Epoch: big.NewInt(0), + EIP155Epoch: big.NewInt(0), + S3Epoch: big.NewInt(0), + DataCopyFixEpoch: big.NewInt(0), + IstanbulEpoch: big.NewInt(0), + ReceiptLogEpoch: big.NewInt(0), + SHA3Epoch: big.NewInt(0), + HIP6And8Epoch: big.NewInt(0), + StakingPrecompileEpoch: big.NewInt(2), + ChainIdFixEpoch: big.NewInt(0), + SlotsLimitedEpoch: EpochTBD, // epoch to enable HIP-16 + CrossShardXferPrecompileEpoch: big.NewInt(1), + AllowlistEpoch: EpochTBD, + FeeCollectEpoch: big.NewInt(574), + LeaderRotationExternalNonBeaconLeaders: EpochTBD, + LeaderRotationExternalBeaconLeaders: EpochTBD, + ValidatorCodeFixEpoch: EpochTBD, } // StressnetChainConfig contains the chain parameters for the Stress test network. // All features except for CrossLink are enabled at launch. StressnetChainConfig = &ChainConfig{ - ChainID: StressnetChainID, - EthCompatibleChainID: EthStressnetShard0ChainID, - EthCompatibleShard0ChainID: EthStressnetShard0ChainID, - EthCompatibleEpoch: big.NewInt(0), - CrossTxEpoch: big.NewInt(0), - CrossLinkEpoch: big.NewInt(2), - AggregatedRewardEpoch: big.NewInt(3), - StakingEpoch: big.NewInt(2), - PreStakingEpoch: big.NewInt(1), - QuickUnlockEpoch: big.NewInt(0), - FiveSecondsEpoch: big.NewInt(0), - TwoSecondsEpoch: big.NewInt(0), - SixtyPercentEpoch: big.NewInt(10), - RedelegationEpoch: big.NewInt(0), - NoEarlyUnlockEpoch: big.NewInt(0), - VRFEpoch: big.NewInt(0), - PrevVRFEpoch: big.NewInt(0), - MinDelegation100Epoch: big.NewInt(0), - MinCommissionRateEpoch: big.NewInt(0), - MinCommissionPromoPeriod: big.NewInt(10), - EPoSBound35Epoch: big.NewInt(0), - EIP155Epoch: big.NewInt(0), - S3Epoch: big.NewInt(0), - DataCopyFixEpoch: big.NewInt(0), - IstanbulEpoch: big.NewInt(0), - ReceiptLogEpoch: big.NewInt(0), - SHA3Epoch: big.NewInt(0), - HIP6And8Epoch: big.NewInt(0), - StakingPrecompileEpoch: big.NewInt(2), - ChainIdFixEpoch: big.NewInt(0), - SlotsLimitedEpoch: EpochTBD, // epoch to enable HIP-16 - CrossShardXferPrecompileEpoch: big.NewInt(1), - AllowlistEpoch: EpochTBD, - FeeCollectEpoch: EpochTBD, - LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 64, - ValidatorCodeFixEpoch: EpochTBD, + ChainID: StressnetChainID, + EthCompatibleChainID: EthStressnetShard0ChainID, + EthCompatibleShard0ChainID: EthStressnetShard0ChainID, + EthCompatibleEpoch: big.NewInt(0), + CrossTxEpoch: big.NewInt(0), + CrossLinkEpoch: big.NewInt(2), + AggregatedRewardEpoch: big.NewInt(3), + StakingEpoch: big.NewInt(2), + PreStakingEpoch: big.NewInt(1), + QuickUnlockEpoch: big.NewInt(0), + FiveSecondsEpoch: big.NewInt(0), + TwoSecondsEpoch: big.NewInt(0), + SixtyPercentEpoch: big.NewInt(10), + RedelegationEpoch: big.NewInt(0), + NoEarlyUnlockEpoch: big.NewInt(0), + VRFEpoch: big.NewInt(0), + PrevVRFEpoch: big.NewInt(0), + MinDelegation100Epoch: big.NewInt(0), + MinCommissionRateEpoch: big.NewInt(0), + MinCommissionPromoPeriod: big.NewInt(10), + EPoSBound35Epoch: big.NewInt(0), + EIP155Epoch: big.NewInt(0), + S3Epoch: big.NewInt(0), + DataCopyFixEpoch: big.NewInt(0), + IstanbulEpoch: big.NewInt(0), + ReceiptLogEpoch: big.NewInt(0), + SHA3Epoch: big.NewInt(0), + HIP6And8Epoch: big.NewInt(0), + StakingPrecompileEpoch: big.NewInt(2), + ChainIdFixEpoch: big.NewInt(0), + SlotsLimitedEpoch: EpochTBD, // epoch to enable HIP-16 + CrossShardXferPrecompileEpoch: big.NewInt(1), + AllowlistEpoch: EpochTBD, + FeeCollectEpoch: EpochTBD, + LeaderRotationExternalNonBeaconLeaders: EpochTBD, + LeaderRotationExternalBeaconLeaders: EpochTBD, + ValidatorCodeFixEpoch: EpochTBD, } // LocalnetChainConfig contains the chain parameters to run for local development. LocalnetChainConfig = &ChainConfig{ - ChainID: TestnetChainID, - EthCompatibleChainID: EthTestnetShard0ChainID, - EthCompatibleShard0ChainID: EthTestnetShard0ChainID, - EthCompatibleEpoch: big.NewInt(0), - CrossTxEpoch: big.NewInt(0), - CrossLinkEpoch: big.NewInt(2), - AggregatedRewardEpoch: big.NewInt(3), - StakingEpoch: big.NewInt(2), - PreStakingEpoch: big.NewInt(0), - QuickUnlockEpoch: big.NewInt(0), - FiveSecondsEpoch: big.NewInt(0), - TwoSecondsEpoch: big.NewInt(0), - SixtyPercentEpoch: EpochTBD, // Never enable it for localnet as localnet has no external validator setup - RedelegationEpoch: big.NewInt(0), - NoEarlyUnlockEpoch: big.NewInt(0), - VRFEpoch: big.NewInt(0), - PrevVRFEpoch: big.NewInt(0), - MinDelegation100Epoch: big.NewInt(0), - MinCommissionRateEpoch: big.NewInt(0), - MinCommissionPromoPeriod: big.NewInt(10), - EPoSBound35Epoch: big.NewInt(0), - EIP155Epoch: big.NewInt(0), - S3Epoch: big.NewInt(0), - DataCopyFixEpoch: big.NewInt(0), - IstanbulEpoch: big.NewInt(0), - ReceiptLogEpoch: big.NewInt(0), - SHA3Epoch: big.NewInt(0), - HIP6And8Epoch: EpochTBD, // Never enable it for localnet as localnet has no external validator setup - StakingPrecompileEpoch: big.NewInt(2), - ChainIdFixEpoch: big.NewInt(0), - SlotsLimitedEpoch: EpochTBD, // epoch to enable HIP-16 - CrossShardXferPrecompileEpoch: big.NewInt(1), - AllowlistEpoch: EpochTBD, - LeaderRotationEpoch: EpochTBD, - LeaderRotationBlocksCount: 5, - FeeCollectEpoch: big.NewInt(5), - ValidatorCodeFixEpoch: EpochTBD, + ChainID: TestnetChainID, + EthCompatibleChainID: EthTestnetShard0ChainID, + EthCompatibleShard0ChainID: EthTestnetShard0ChainID, + EthCompatibleEpoch: big.NewInt(0), + CrossTxEpoch: big.NewInt(0), + CrossLinkEpoch: big.NewInt(2), + AggregatedRewardEpoch: big.NewInt(3), + StakingEpoch: big.NewInt(2), + PreStakingEpoch: big.NewInt(0), + QuickUnlockEpoch: big.NewInt(0), + FiveSecondsEpoch: big.NewInt(0), + TwoSecondsEpoch: big.NewInt(0), + SixtyPercentEpoch: EpochTBD, // Never enable it for localnet as localnet has no external validator setup + RedelegationEpoch: big.NewInt(0), + NoEarlyUnlockEpoch: big.NewInt(0), + VRFEpoch: big.NewInt(0), + PrevVRFEpoch: big.NewInt(0), + MinDelegation100Epoch: big.NewInt(0), + MinCommissionRateEpoch: big.NewInt(0), + MinCommissionPromoPeriod: big.NewInt(10), + EPoSBound35Epoch: big.NewInt(0), + EIP155Epoch: big.NewInt(0), + S3Epoch: big.NewInt(0), + DataCopyFixEpoch: big.NewInt(0), + IstanbulEpoch: big.NewInt(0), + ReceiptLogEpoch: big.NewInt(0), + SHA3Epoch: big.NewInt(0), + HIP6And8Epoch: EpochTBD, // Never enable it for localnet as localnet has no external validator setup + StakingPrecompileEpoch: big.NewInt(2), + ChainIdFixEpoch: big.NewInt(0), + SlotsLimitedEpoch: EpochTBD, // epoch to enable HIP-16 + CrossShardXferPrecompileEpoch: big.NewInt(1), + AllowlistEpoch: EpochTBD, + LeaderRotationExternalNonBeaconLeaders: big.NewInt(5), + LeaderRotationExternalBeaconLeaders: big.NewInt(6), + FeeCollectEpoch: big.NewInt(5), + ValidatorCodeFixEpoch: EpochTBD, } // AllProtocolChanges ... @@ -319,8 +319,8 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount + big.NewInt(1), // LeaderRotationExternalNonBeaconLeaders + big.NewInt(1), // LeaderRotationExternalBeaconLeaders big.NewInt(0), // FeeCollectEpoch big.NewInt(0), // ValidatorCodeFixEpoch } @@ -362,8 +362,8 @@ var ( big.NewInt(0), // SlotsLimitedEpoch big.NewInt(1), // CrossShardXferPrecompileEpoch big.NewInt(0), // AllowlistEpoch - big.NewInt(1), // LeaderRotationEpoch - 64, // LeaderRotationBlocksCount + big.NewInt(1), // LeaderRotationExternalNonBeaconLeaders + big.NewInt(1), // LeaderRotationExternalBeaconLeaders big.NewInt(0), // FeeCollectEpoch big.NewInt(0), // ValidatorCodeFixEpoch } @@ -505,9 +505,9 @@ type ChainConfig struct { // AllowlistEpoch is the first epoch to support allowlist of HIP18 AllowlistEpoch *big.Int - LeaderRotationEpoch *big.Int `json:"leader-rotation-epoch,omitempty"` + LeaderRotationExternalNonBeaconLeaders *big.Int `json:"leader-rotation-external-non-beacon-leaders,omitempty"` - LeaderRotationBlocksCount int `json:"leader-rotation-blocks-count,omitempty"` + LeaderRotationExternalBeaconLeaders *big.Int `json:"leader-rotation-external-beacon-leaders,omitempty"` // FeeCollectEpoch is the first epoch that enables txn fees to be collected into the community-managed account. // It should >= StakingEpoch. @@ -725,7 +725,17 @@ func (c *ChainConfig) IsAllowlistEpoch(epoch *big.Int) bool { } func (c *ChainConfig) IsLeaderRotation(epoch *big.Int) bool { - return isForked(c.LeaderRotationEpoch, epoch) + return isForked(c.LeaderRotationExternalNonBeaconLeaders, epoch) +} + +func (c *ChainConfig) IsLeaderRotationExternalValidatorsAllowed(epoch *big.Int, shardID uint32) bool { + if !c.IsLeaderRotation(epoch) { + return false + } + if shardID == 0 { + return isForked(c.LeaderRotationExternalBeaconLeaders, epoch) + } + return true } // IsFeeCollectEpoch determines whether Txn Fees will be collected into the community-managed account. diff --git a/numeric/decimal.go b/numeric/decimal.go index e4236daa46..f80c9eca14 100644 --- a/numeric/decimal.go +++ b/numeric/decimal.go @@ -202,6 +202,12 @@ func (d Dec) Copy() Dec { } } +func (d Dec) Div(d2 Dec) Dec { + return Dec{ + new(big.Int).Div(d.Int, d2.Int), + } +} + // IsNil ... func (d Dec) IsNil() bool { return d.Int == nil } // is decimal nil // IsZero ... diff --git a/numeric/decimal_test.go b/numeric/decimal_test.go index a7937d9a70..7b790f20de 100644 --- a/numeric/decimal_test.go +++ b/numeric/decimal_test.go @@ -368,3 +368,27 @@ func TestDecCeil(t *testing.T) { require.Equal(t, tc.expected, res, "unexpected result for test case %d, input: %v", i, tc.input) } } + +func TestDiv(t *testing.T) { + tests := []struct { + d1, d2, exp Dec + }{ + {mustNewDecFromStr(t, "0"), mustNewDecFromStr(t, "1"), ZeroDec()}, + {mustNewDecFromStr(t, "1"), mustNewDecFromStr(t, "1"), NewDec(1)}, + {mustNewDecFromStr(t, "1"), mustNewDecFromStr(t, "2"), mustNewDecFromStr(t, "0.5")}, + {mustNewDecFromStr(t, "2"), mustNewDecFromStr(t, "1"), NewDec(2)}, + {mustNewDecFromStr(t, "2"), mustNewDecFromStr(t, "3"), mustNewDecFromStr(t, "0.666666666666666667")}, + {mustNewDecFromStr(t, "2"), mustNewDecFromStr(t, "4"), mustNewDecFromStr(t, "0.5")}, + {mustNewDecFromStr(t, "2"), mustNewDecFromStr(t, "5"), mustNewDecFromStr(t, "0.4")}, + {mustNewDecFromStr(t, "2"), mustNewDecFromStr(t, "6"), mustNewDecFromStr(t, "0.333333333333333333")}, + {mustNewDecFromStr(t, "2"), mustNewDecFromStr(t, "7"), mustNewDecFromStr(t, "0.285714285714285714")}, + {mustNewDecFromStr(t, "2"), mustNewDecFromStr(t, "8"), mustNewDecFromStr(t, "0.25")}, + {mustNewDecFromStr(t, "2"), mustNewDecFromStr(t, "9"), mustNewDecFromStr(t, "0.222222222222222222")}, + {mustNewDecFromStr(t, "2"), mustNewDecFromStr(t, "10"), mustNewDecFromStr(t, "0.2")}, + } + for i, tc := range tests { + res := tc.d1.Quo(tc.d2) + require.True(t, res.Equal(tc.exp), "unexpected result for test case %d, input: %s %s %s", i, tc.d1, tc.d2, tc.exp) + } + +} From 53161ca33c3bb5e2c4dc5da2faca9e9af4a3a84d Mon Sep 17 00:00:00 2001 From: Gheis Mohammadi Date: Tue, 11 Apr 2023 23:56:34 +0000 Subject: [PATCH 390/420] Add prefix for validator wrapper (#4402) * add separate prefix for validator wrapper * update comments * make read/write backward compatible * add validator codes to stats * goimports * goimports accessor_state --- core/genesis.go | 2 +- core/rawdb/accessors_state.go | 54 ++++++++++++++++++++++++++ core/rawdb/database.go | 4 ++ core/rawdb/schema.go | 23 +++++++++-- core/state/database.go | 47 ++++++++++++++++++++++ core/state/dump.go | 2 +- core/state/iterator.go | 6 +++ core/state/iterator_test.go | 2 +- core/state/journal.go | 2 +- core/state/prefeth.go | 2 +- core/state/state_object.go | 43 ++++++++++++++------ core/state/state_test.go | 8 ++-- core/state/statedb.go | 27 ++++++++----- core/state/statedb_test.go | 44 ++++++++++----------- core/state/trie_prefetcher_test.go | 6 +-- core/vm/contracts_write.go | 2 +- core/vm/evm.go | 10 ++--- core/vm/gas_table_test.go | 2 +- core/vm/instructions.go | 4 +- core/vm/interface.go | 6 +-- core/vm/logger_test.go | 8 ++-- core/vm/runtime/runtime.go | 2 +- core/vm/runtime/runtime_test.go | 4 +- hmy/tracers/block_tracer.go | 2 +- hmy/tracers/tracer.go | 2 +- rosetta/services/block.go | 4 +- rosetta/services/construction_check.go | 2 +- rpc/contract.go | 2 +- staking/availability/measure_test.go | 6 +-- 29 files changed, 241 insertions(+), 87 deletions(-) diff --git a/core/genesis.go b/core/genesis.go index 3bd4f718aa..3e230cf9e6 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -247,7 +247,7 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { statedb, _ := state.New(common.Hash{}, state.NewDatabase(db), nil) for addr, account := range g.Alloc { statedb.AddBalance(addr, account.Balance) - statedb.SetCode(addr, account.Code) + statedb.SetCode(addr, account.Code, false) statedb.SetNonce(addr, account.Nonce) for key, value := range account.Storage { statedb.SetState(addr, key, value) diff --git a/core/rawdb/accessors_state.go b/core/rawdb/accessors_state.go index 351928fcd5..72dbe94fb6 100644 --- a/core/rawdb/accessors_state.go +++ b/core/rawdb/accessors_state.go @@ -93,3 +93,57 @@ func DeleteCode(db ethdb.KeyValueWriter, hash common.Hash) { utils.Logger().Error().Err(err).Msg("Failed to delete contract code") } } + +// ReadValidatorCode retrieves the validator code of the provided code hash. +func ReadValidatorCode(db ethdb.KeyValueReader, hash common.Hash) []byte { + // Try with the prefixed code scheme first, if not then try with legacy + // scheme. + data := ReadValidatorCodeWithPrefix(db, hash) + if len(data) != 0 { + return data + } + data, _ = db.Get(hash.Bytes()) + return data +} + +// ReadValidatorCodeWithPrefix retrieves the validator code of the provided code hash. +// The main difference between this function and ReadValidatorCode is this function +// will only check the existence with latest scheme(with prefix). +func ReadValidatorCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) []byte { + data, _ := db.Get(validatorCodeKey(hash)) + return data +} + +// HasValidatorCode checks if the validator code corresponding to the +// provided code hash is present in the db. +func HasValidatorCode(db ethdb.KeyValueReader, hash common.Hash) bool { + // Try with the prefixed code scheme first, if not then try with legacy + // scheme. + if ok := HasValidatorCodeWithPrefix(db, hash); ok { + return true + } + ok, _ := db.Has(hash.Bytes()) + return ok +} + +// HasValidatorCodeWithPrefix checks if the validator code corresponding to the +// provided code hash is present in the db. This function will only check +// presence using the prefix-scheme. +func HasValidatorCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) bool { + ok, _ := db.Has(validatorCodeKey(hash)) + return ok +} + +// WriteValidatorCode writes the provided validator code to database. +func WriteValidatorCode(db ethdb.KeyValueWriter, hash common.Hash, code []byte) { + if err := db.Put(validatorCodeKey(hash), code); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to store validator code") + } +} + +// DeleteValidatorCode deletes the specified validator code from the database. +func DeleteValidatorCode(db ethdb.KeyValueWriter, hash common.Hash) { + if err := db.Delete(validatorCodeKey(hash)); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to delete validator code") + } +} diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 5cdae00e38..c1421b5073 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -316,6 +316,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { hashNumPairings stat tries stat codes stat + validatorCodes stat txLookups stat accountSnaps stat storageSnaps stat @@ -359,6 +360,8 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { tries.Add(size) case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength: codes.Add(size) + case bytes.HasPrefix(key, ValidatorCodePrefix) && len(key) == len(ValidatorCodePrefix)+common.HashLength: + validatorCodes.Add(size) case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): txLookups.Add(size) case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength): @@ -422,6 +425,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { {"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()}, {"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()}, {"Key-Value store", "Contract codes", codes.Size(), codes.Count()}, + {"Key-Value store", "Validator codes", validatorCodes.Size(), validatorCodes.Count()}, {"Key-Value store", "Trie nodes", tries.Size(), tries.Count()}, {"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()}, {"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()}, diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 798f27f59e..56147b51d3 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -126,10 +126,11 @@ var ( snapdbInfoKey = []byte("SnapdbInfo") // Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes). - SnapshotAccountPrefix = []byte("a") // SnapshotAccountPrefix + account hash -> account trie value - SnapshotStoragePrefix = []byte("o") // SnapshotStoragePrefix + account hash + storage hash -> storage trie value - CodePrefix = []byte("c") // CodePrefix + code hash -> account code -> We not using this at the moment - skeletonHeaderPrefix = []byte("S") // skeletonHeaderPrefix + num (uint64 big endian) -> header + SnapshotAccountPrefix = []byte("a") // SnapshotAccountPrefix + account hash -> account trie value + SnapshotStoragePrefix = []byte("o") // SnapshotStoragePrefix + account hash + storage hash -> storage trie value + CodePrefix = []byte("c") // CodePrefix + code hash -> account code + ValidatorCodePrefix = []byte("vc") // ValidatorCodePrefix + code hash -> validator code + skeletonHeaderPrefix = []byte("S") // skeletonHeaderPrefix + num (uint64 big endian) -> header // Path-based trie node scheme. trieNodeAccountPrefix = []byte("A") // trieNodeAccountPrefix + hexPath -> trie node @@ -196,6 +197,20 @@ func IsCodeKey(key []byte) (bool, []byte) { return false, nil } +// validatorCodeKey = ValidatorCodePrefix + hash +func validatorCodeKey(hash common.Hash) []byte { + return append(ValidatorCodePrefix, hash.Bytes()...) +} + +// IsValidatorCodeKey reports whether the given byte slice is the key of validator code, +// if so return the raw code hash as well. +func IsValidatorCodeKey(key []byte) (bool, []byte) { + if bytes.HasPrefix(key, ValidatorCodePrefix) && len(key) == common.HashLength+len(ValidatorCodePrefix) { + return true, key[len(ValidatorCodePrefix):] + } + return false, nil +} + // genesisStateSpecKey = genesisPrefix + hash func genesisStateSpecKey(hash common.Hash) []byte { return append(genesisPrefix, hash.Bytes()...) diff --git a/core/state/database.go b/core/state/database.go index d6a0e7e15b..c1e375ccde 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -53,6 +53,12 @@ type Database interface { // ContractCodeSize retrieves a particular contracts code's size. ContractCodeSize(addrHash, codeHash common.Hash) (int, error) + // ValidatorCode retrieves a particular validator's code. + ValidatorCode(addrHash, codeHash common.Hash) ([]byte, error) + + // ValidatorCodeSize retrieves a particular validator code's size. + ValidatorCodeSize(addrHash, codeHash common.Hash) (int, error) + // DiskDB returns the underlying key-value disk database. DiskDB() ethdb.KeyValueStore @@ -234,6 +240,47 @@ func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, erro return len(code), err } +// ValidatorCodeSize retrieves a particular validators code's size. +func (db *cachingDB) ValidatorCodeSize(addrHash, codeHash common.Hash) (int, error) { + if cached, ok := db.codeSizeCache.Get(codeHash); ok { + return cached, nil + } + code, err := db.ValidatorCode(addrHash, codeHash) + return len(code), err +} + +// ValidatorCode retrieves a particular validator's code. +func (db *cachingDB) ValidatorCode(addrHash, codeHash common.Hash) ([]byte, error) { + code, _ := db.codeCache.Get(codeHash) + if len(code) > 0 { + return code, nil + } + code = rawdb.ReadValidatorCode(db.disk, codeHash) + if len(code) > 0 { + db.codeCache.Add(codeHash, code) + db.codeSizeCache.Add(codeHash, len(code)) + return code, nil + } + return nil, errors.New("not found") +} + +// ValidatorCodeWithPrefix retrieves a particular validator's code. If the +// code can't be found in the cache, then check the existence with **new** +// db scheme. +func (db *cachingDB) ValidatorCodeWithPrefix(addrHash, codeHash common.Hash) ([]byte, error) { + code, _ := db.codeCache.Get(codeHash) + if len(code) > 0 { + return code, nil + } + code = rawdb.ReadValidatorCodeWithPrefix(db.disk, codeHash) + if len(code) > 0 { + db.codeCache.Add(codeHash, code) + db.codeSizeCache.Add(codeHash, len(code)) + return code, nil + } + return nil, errors.New("not found") +} + // DiskDB returns the underlying key-value disk database. func (db *cachingDB) DiskDB() ethdb.KeyValueStore { return db.disk diff --git a/core/state/dump.go b/core/state/dump.go index d9e031ed4a..c4d70f6e00 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -165,7 +165,7 @@ func (s *DB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []byte) addr := common.BytesToAddress(addrBytes) obj := newObject(s, addr, data) if !conf.SkipCode { - account.Code = obj.Code(s.db) + account.Code = obj.Code(s.db, false) } if !conf.SkipStorage { account.Storage = make(map[common.Hash]string) diff --git a/core/state/iterator.go b/core/state/iterator.go index dea2c81b03..8bc26332dd 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -124,6 +124,12 @@ func (it *NodeIterator) step() error { if err != nil { return fmt.Errorf("code %x: %v", account.CodeHash, err) } + if it.code == nil || len(it.code) == 0 { + it.code, err = it.state.db.ValidatorCode(addrHash, common.BytesToHash(account.CodeHash)) + if err != nil { + return fmt.Errorf("code %x: %v", account.CodeHash, err) + } + } } it.accountHash = it.stateIt.Parent() return nil diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go index 50374059b7..8333dc8bce 100644 --- a/core/state/iterator_test.go +++ b/core/state/iterator_test.go @@ -54,7 +54,7 @@ func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) { acc.nonce = uint64(42 * i) if i%3 == 0 { - obj.SetCode(crypto.Keccak256Hash([]byte{i, i, i, i, i}), []byte{i, i, i, i, i}) + obj.SetCode(crypto.Keccak256Hash([]byte{i, i, i, i, i}), []byte{i, i, i, i, i}, false) acc.code = []byte{i, i, i, i, i} } if i%5 == 0 { diff --git a/core/state/journal.go b/core/state/journal.go index 48dd66f665..210f38808f 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -217,7 +217,7 @@ func (ch nonceChange) dirtied() *common.Address { } func (ch codeChange) revert(s *DB) { - s.getStateObject(*ch.account).setCode(common.BytesToHash(ch.prevhash), ch.prevcode) + s.getStateObject(*ch.account).setCode(common.BytesToHash(ch.prevhash), ch.prevcode, false) } func (ch codeChange) dirtied() *common.Address { diff --git a/core/state/prefeth.go b/core/state/prefeth.go index 4d3b317bb1..0b19f80d74 100644 --- a/core/state/prefeth.go +++ b/core/state/prefeth.go @@ -100,7 +100,7 @@ func (s *DB) prefetchWorker(job *prefetchJob, jobs chan *prefetchJob) { addr := common.BytesToAddress(addrBytes) obj := newObject(s, addr, data) if data.CodeHash != nil { - obj.Code(s.db) + obj.Code(s.db, false) } // build account trie tree diff --git a/core/state/state_object.go b/core/state/state_object.go index a235b18f73..262f5136ee 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -93,9 +93,10 @@ type Object struct { // Cache flags. // When an object is marked suicided it will be delete from the trie // during the "update" phase of the state transition. - dirtyCode bool // true if the code was updated - suicided bool - deleted bool + validatorWrapper bool // true if the code belongs to validator wrapper + dirtyCode bool // true if the code was updated + suicided bool + deleted bool } // empty returns whether the account is considered empty. @@ -497,14 +498,24 @@ func (s *Object) Address() common.Address { return s.address } -// Code returns the contract code associated with this object, if any. -func (s *Object) Code(db Database) []byte { +// Code returns the contract/validator code associated with this object, if any. +func (s *Object) Code(db Database, isValidatorCode bool) []byte { if s.code != nil { return s.code } if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) { return nil } + if s.validatorWrapper || isValidatorCode { + code, err := db.ValidatorCode(s.addrHash, common.BytesToHash(s.CodeHash())) + if err != nil { + s.setError(fmt.Errorf("can't load validator code hash %x: %v", s.CodeHash(), err)) + } + if code != nil { + s.code = code + return code + } + } code, err := db.ContractCode(s.addrHash, common.BytesToHash(s.CodeHash())) if err != nil { s.setError(fmt.Errorf("can't load code hash %x: %v", s.CodeHash(), err)) @@ -513,16 +524,25 @@ func (s *Object) Code(db Database) []byte { return code } -// CodeSize returns the size of the contract code associated with this object, +// CodeSize returns the size of the contract/validator code associated with this object, // or zero if none. This method is an almost mirror of Code, but uses a cache // inside the database to avoid loading codes seen recently. -func (s *Object) CodeSize(db Database) int { +func (s *Object) CodeSize(db Database, isValidatorCode bool) int { if s.code != nil { return len(s.code) } if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) { return 0 } + if s.validatorWrapper || isValidatorCode { + size, err := db.ValidatorCodeSize(s.addrHash, common.BytesToHash(s.CodeHash())) + if err != nil { + s.setError(fmt.Errorf("can't load validator code size %x: %v", s.CodeHash(), err)) + } + if size > 0 { + return size + } + } size, err := db.ContractCodeSize(s.addrHash, common.BytesToHash(s.CodeHash())) if err != nil { s.setError(fmt.Errorf("can't load code size %x: %v", s.CodeHash(), err)) @@ -530,20 +550,21 @@ func (s *Object) CodeSize(db Database) int { return size } -func (s *Object) SetCode(codeHash common.Hash, code []byte) { - prevcode := s.Code(s.db.db) +func (s *Object) SetCode(codeHash common.Hash, code []byte, isValidatorCode bool) { + prevcode := s.Code(s.db.db, isValidatorCode) s.db.journal.append(codeChange{ account: &s.address, prevhash: s.CodeHash(), prevcode: prevcode, }) - s.setCode(codeHash, code) + s.setCode(codeHash, code, isValidatorCode) } -func (s *Object) setCode(codeHash common.Hash, code []byte) { +func (s *Object) setCode(codeHash common.Hash, code []byte, isValidatorCode bool) { s.code = code s.data.CodeHash = codeHash[:] s.dirtyCode = true + s.validatorWrapper = isValidatorCode } func (s *Object) SetNonce(nonce uint64) { diff --git a/core/state/state_test.go b/core/state/state_test.go index 9209d3de29..9d209a80be 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -48,7 +48,7 @@ func TestDump(t *testing.T) { obj1 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x01})) obj1.AddBalance(big.NewInt(22)) obj2 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x01, 0x02})) - obj2.SetCode(crypto.Keccak256Hash([]byte{3, 3, 3, 3, 3, 3, 3}), []byte{3, 3, 3, 3, 3, 3, 3}) + obj2.SetCode(crypto.Keccak256Hash([]byte{3, 3, 3, 3, 3, 3, 3}), []byte{3, 3, 3, 3, 3, 3, 3}, false) obj3 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x02})) obj3.SetBalance(big.NewInt(44)) @@ -166,7 +166,7 @@ func TestSnapshot2(t *testing.T) { so0 := state.getStateObject(stateobjaddr0) so0.SetBalance(big.NewInt(42)) so0.SetNonce(43) - so0.SetCode(crypto.Keccak256Hash([]byte{'c', 'a', 'f', 'e'}), []byte{'c', 'a', 'f', 'e'}) + so0.SetCode(crypto.Keccak256Hash([]byte{'c', 'a', 'f', 'e'}), []byte{'c', 'a', 'f', 'e'}, false) so0.suicided = false so0.deleted = false state.setStateObject(so0) @@ -178,7 +178,7 @@ func TestSnapshot2(t *testing.T) { so1 := state.getStateObject(stateobjaddr1) so1.SetBalance(big.NewInt(52)) so1.SetNonce(53) - so1.SetCode(crypto.Keccak256Hash([]byte{'c', 'a', 'f', 'e', '2'}), []byte{'c', 'a', 'f', 'e', '2'}) + so1.SetCode(crypto.Keccak256Hash([]byte{'c', 'a', 'f', 'e', '2'}), []byte{'c', 'a', 'f', 'e', '2'}, false) so1.suicided = true so1.deleted = true state.setStateObject(so1) @@ -194,7 +194,7 @@ func TestSnapshot2(t *testing.T) { so0Restored := state.getStateObject(stateobjaddr0) // Update lazily-loaded values before comparing. so0Restored.GetState(state.db, storageaddr) - so0Restored.Code(state.db) + so0Restored.Code(state.db, false) // non-deleted is equal (restored) compareStateObjects(so0Restored, so0, t) diff --git a/core/state/statedb.go b/core/state/statedb.go index eff2a37c2a..30692e4d44 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -342,18 +342,18 @@ func (db *DB) BlockHash() common.Hash { return db.bhash } -func (db *DB) GetCode(addr common.Address) []byte { +func (db *DB) GetCode(addr common.Address, isValidatorCode bool) []byte { Object := db.getStateObject(addr) if Object != nil { - return Object.Code(db.db) + return Object.Code(db.db, isValidatorCode) } return nil } -func (db *DB) GetCodeSize(addr common.Address) int { +func (db *DB) GetCodeSize(addr common.Address, isValidatorCode bool) int { Object := db.getStateObject(addr) if Object != nil { - return Object.CodeSize(db.db) + return Object.CodeSize(db.db, isValidatorCode) } return 0 } @@ -475,10 +475,10 @@ func (db *DB) SetNonce(addr common.Address, nonce uint64) { } } -func (db *DB) SetCode(addr common.Address, code []byte) { +func (db *DB) SetCode(addr common.Address, code []byte, isValidatorCode bool) { Object := db.GetOrNewStateObject(addr) if Object != nil { - Object.SetCode(crypto.Keccak256Hash(code), code) + Object.SetCode(crypto.Keccak256Hash(code), code, isValidatorCode) } } @@ -1053,7 +1053,11 @@ func (db *DB) Commit(deleteEmptyObjects bool) (common.Hash, error) { if obj := db.stateObjects[addr]; !obj.deleted { // Write any contract code associated with the state object if obj.code != nil && obj.dirtyCode { - rawdb.WriteCode(codeWriter, common.BytesToHash(obj.CodeHash()), obj.code) + if obj.validatorWrapper { + rawdb.WriteValidatorCode(codeWriter, common.BytesToHash(obj.CodeHash()), obj.code) + } else { + rawdb.WriteCode(codeWriter, common.BytesToHash(obj.CodeHash()), obj.code) + } obj.dirtyCode = false } // Write any storage changes in the state object to its storage trie @@ -1237,9 +1241,12 @@ func (db *DB) ValidatorWrapper( return copyValidatorWrapperIfNeeded(cached, sendOriginal, copyDelegations), nil } - by := db.GetCode(addr) + by := db.GetCode(addr, true) if len(by) == 0 { - return nil, ErrAddressNotPresent + by = db.GetCode(addr, false) + if len(by) == 0 { + return nil, ErrAddressNotPresent + } } val := stk.ValidatorWrapper{} if err := rlp.DecodeBytes(by, &val); err != nil { @@ -1285,7 +1292,7 @@ func (db *DB) UpdateValidatorWrapper( return err } // has revert in-built for the code field - db.SetCode(addr, by) + db.SetCode(addr, by, true) // update cache db.stateValidators[addr] = val return nil diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index db2cf5ebfd..f4277809bd 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -50,7 +50,7 @@ func TestUpdateLeaks(t *testing.T) { state.SetState(addr, common.BytesToHash([]byte{i, i, i}), common.BytesToHash([]byte{i, i, i, i})) } if i%3 == 0 { - state.SetCode(addr, []byte{i, i, i, i, i}) + state.SetCode(addr, []byte{i, i, i, i, i}, false) } } @@ -84,7 +84,7 @@ func TestIntermediateLeaks(t *testing.T) { state.SetState(addr, common.Hash{i, i, i, tweak}, common.Hash{i, i, i, i, tweak}) } if i%3 == 0 { - state.SetCode(addr, []byte{i, i, i, i, i, tweak}) + state.SetCode(addr, []byte{i, i, i, i, i, tweak}, false) } } @@ -286,7 +286,7 @@ func newTestAction(addr common.Address, r *rand.Rand) testAction { code := make([]byte, 16) binary.BigEndian.PutUint64(code, uint64(a.args[0])) binary.BigEndian.PutUint64(code[8:], uint64(a.args[1])) - s.SetCode(addr, code) + s.SetCode(addr, code, false) }, args: make([]int64, 2), }, @@ -452,9 +452,9 @@ func (test *snapshotTest) checkEqual(state, checkstate *DB) error { checkeq("HasSuicided", state.HasSuicided(addr), checkstate.HasSuicided(addr)) checkeq("GetBalance", state.GetBalance(addr), checkstate.GetBalance(addr)) checkeq("GetNonce", state.GetNonce(addr), checkstate.GetNonce(addr)) - checkeq("GetCode", state.GetCode(addr), checkstate.GetCode(addr)) + checkeq("GetCode", state.GetCode(addr, false), checkstate.GetCode(addr, false)) checkeq("GetCodeHash", state.GetCodeHash(addr), checkstate.GetCodeHash(addr)) - checkeq("GetCodeSize", state.GetCodeSize(addr), checkstate.GetCodeSize(addr)) + checkeq("GetCodeSize", state.GetCodeSize(addr, false), checkstate.GetCodeSize(addr, false)) // Check storage. if obj := state.getStateObject(addr); obj != nil { state.ForEachStorage(addr, func(key, value common.Hash) bool { @@ -525,14 +525,14 @@ func TestCopyCommitCopy(t *testing.T) { skey := common.HexToHash("aaa") sval := common.HexToHash("bbb") - state.SetBalance(addr, big.NewInt(42)) // Change the account trie - state.SetCode(addr, []byte("hello")) // Change an external metadata - state.SetState(addr, skey, sval) // Change the storage trie + state.SetBalance(addr, big.NewInt(42)) // Change the account trie + state.SetCode(addr, []byte("hello"), false) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie if balance := state.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("initial balance mismatch: have %v, want %v", balance, 42) } - if code := state.GetCode(addr); !bytes.Equal(code, []byte("hello")) { + if code := state.GetCode(addr, false); !bytes.Equal(code, []byte("hello")) { t.Fatalf("initial code mismatch: have %x, want %x", code, []byte("hello")) } if val := state.GetState(addr, skey); val != sval { @@ -546,7 +546,7 @@ func TestCopyCommitCopy(t *testing.T) { if balance := copyOne.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("first copy pre-commit balance mismatch: have %v, want %v", balance, 42) } - if code := copyOne.GetCode(addr); !bytes.Equal(code, []byte("hello")) { + if code := copyOne.GetCode(addr, false); !bytes.Equal(code, []byte("hello")) { t.Fatalf("first copy pre-commit code mismatch: have %x, want %x", code, []byte("hello")) } if val := copyOne.GetState(addr, skey); val != sval { @@ -560,7 +560,7 @@ func TestCopyCommitCopy(t *testing.T) { if balance := copyOne.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("first copy post-commit balance mismatch: have %v, want %v", balance, 42) } - if code := copyOne.GetCode(addr); !bytes.Equal(code, []byte("hello")) { + if code := copyOne.GetCode(addr, false); !bytes.Equal(code, []byte("hello")) { t.Fatalf("first copy post-commit code mismatch: have %x, want %x", code, []byte("hello")) } if val := copyOne.GetState(addr, skey); val != sval { @@ -574,7 +574,7 @@ func TestCopyCommitCopy(t *testing.T) { if balance := copyTwo.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("second copy balance mismatch: have %v, want %v", balance, 42) } - if code := copyTwo.GetCode(addr); !bytes.Equal(code, []byte("hello")) { + if code := copyTwo.GetCode(addr, false); !bytes.Equal(code, []byte("hello")) { t.Fatalf("second copy code mismatch: have %x, want %x", code, []byte("hello")) } if val := copyTwo.GetState(addr, skey); val != sval { @@ -597,14 +597,14 @@ func TestCopyCopyCommitCopy(t *testing.T) { skey := common.HexToHash("aaa") sval := common.HexToHash("bbb") - state.SetBalance(addr, big.NewInt(42)) // Change the account trie - state.SetCode(addr, []byte("hello")) // Change an external metadata - state.SetState(addr, skey, sval) // Change the storage trie + state.SetBalance(addr, big.NewInt(42)) // Change the account trie + state.SetCode(addr, []byte("hello"), false) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie if balance := state.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("initial balance mismatch: have %v, want %v", balance, 42) } - if code := state.GetCode(addr); !bytes.Equal(code, []byte("hello")) { + if code := state.GetCode(addr, false); !bytes.Equal(code, []byte("hello")) { t.Fatalf("initial code mismatch: have %x, want %x", code, []byte("hello")) } if val := state.GetState(addr, skey); val != sval { @@ -618,7 +618,7 @@ func TestCopyCopyCommitCopy(t *testing.T) { if balance := copyOne.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("first copy balance mismatch: have %v, want %v", balance, 42) } - if code := copyOne.GetCode(addr); !bytes.Equal(code, []byte("hello")) { + if code := copyOne.GetCode(addr, false); !bytes.Equal(code, []byte("hello")) { t.Fatalf("first copy code mismatch: have %x, want %x", code, []byte("hello")) } if val := copyOne.GetState(addr, skey); val != sval { @@ -632,7 +632,7 @@ func TestCopyCopyCommitCopy(t *testing.T) { if balance := copyTwo.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("second copy pre-commit balance mismatch: have %v, want %v", balance, 42) } - if code := copyTwo.GetCode(addr); !bytes.Equal(code, []byte("hello")) { + if code := copyTwo.GetCode(addr, false); !bytes.Equal(code, []byte("hello")) { t.Fatalf("second copy pre-commit code mismatch: have %x, want %x", code, []byte("hello")) } if val := copyTwo.GetState(addr, skey); val != sval { @@ -645,7 +645,7 @@ func TestCopyCopyCommitCopy(t *testing.T) { if balance := copyTwo.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("second copy post-commit balance mismatch: have %v, want %v", balance, 42) } - if code := copyTwo.GetCode(addr); !bytes.Equal(code, []byte("hello")) { + if code := copyTwo.GetCode(addr, false); !bytes.Equal(code, []byte("hello")) { t.Fatalf("second copy post-commit code mismatch: have %x, want %x", code, []byte("hello")) } if val := copyTwo.GetState(addr, skey); val != sval { @@ -659,7 +659,7 @@ func TestCopyCopyCommitCopy(t *testing.T) { if balance := copyThree.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("third copy balance mismatch: have %v, want %v", balance, 42) } - if code := copyThree.GetCode(addr); !bytes.Equal(code, []byte("hello")) { + if code := copyThree.GetCode(addr, false); !bytes.Equal(code, []byte("hello")) { t.Fatalf("third copy code mismatch: have %x, want %x", code, []byte("hello")) } if val := copyThree.GetState(addr, skey); val != sval { @@ -717,10 +717,10 @@ func TestMissingTrieNodes(t *testing.T) { addr := common.BytesToAddress([]byte("so")) { state.SetBalance(addr, big.NewInt(1)) - state.SetCode(addr, []byte{1, 2, 3}) + state.SetCode(addr, []byte{1, 2, 3}, false) a2 := common.BytesToAddress([]byte("another")) state.SetBalance(a2, big.NewInt(100)) - state.SetCode(a2, []byte{1, 2, 4}) + state.SetCode(a2, []byte{1, 2, 4}, false) root, _ = state.Commit(false) t.Logf("root: %x", root) // force-flush diff --git a/core/state/trie_prefetcher_test.go b/core/state/trie_prefetcher_test.go index ffd67e02a8..d5774d38fd 100644 --- a/core/state/trie_prefetcher_test.go +++ b/core/state/trie_prefetcher_test.go @@ -33,9 +33,9 @@ func filledStateDB() *DB { skey := common.HexToHash("aaa") sval := common.HexToHash("bbb") - state.SetBalance(addr, big.NewInt(42)) // Change the account trie - state.SetCode(addr, []byte("hello")) // Change an external metadata - state.SetState(addr, skey, sval) // Change the storage trie + state.SetBalance(addr, big.NewInt(42)) // Change the account trie + state.SetCode(addr, []byte("hello"), false) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie for i := 0; i < 100; i++ { sk := common.BigToHash(big.NewInt(int64(i))) state.SetState(addr, sk, sk) // Change the storage trie diff --git a/core/vm/contracts_write.go b/core/vm/contracts_write.go index 46ba7fe923..7e24eb9a27 100644 --- a/core/vm/contracts_write.go +++ b/core/vm/contracts_write.go @@ -242,7 +242,7 @@ func (c *crossShardXferPrecompile) RunWriteCapable( return nil, err } // validate not a contract (toAddress can still be a contract) - if len(evm.StateDB.GetCode(fromAddress)) > 0 && !evm.IsValidator(evm.StateDB, fromAddress) { + if len(evm.StateDB.GetCode(fromAddress, false)) > 0 && !evm.IsValidator(evm.StateDB, fromAddress) { return nil, errors.New("cross shard xfer not yet implemented for contracts") } // can't have too many shards diff --git a/core/vm/evm.go b/core/vm/evm.go index 49857255fa..53da390dba 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -336,7 +336,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas evm.Transfer(evm.StateDB, caller.Address(), to.Address(), value, txType) codeHash := evm.StateDB.GetCodeHash(addr) - code := evm.StateDB.GetCode(addr) + code := evm.StateDB.GetCode(addr, false) // If address is a validator address, then it's not a smart contract address // we don't use its code and codeHash fields if evm.Context.IsValidator(evm.StateDB, addr) { @@ -402,7 +402,7 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, // EVM. The contract is a scoped environment for this execution context // only. contract := NewContract(caller, to, value, gas) - contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) + contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr, false)) ret, err = run(evm, contract, input, false) if err != nil { @@ -435,7 +435,7 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by // Initialise a new contract and make initialise the delegate values contract := NewContract(caller, to, nil, gas).AsDelegate() - contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) + contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr, false)) ret, err = run(evm, contract, input, false) if err != nil { @@ -468,7 +468,7 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte // EVM. The contract is a scoped environment for this execution context // only. contract := NewContract(caller, to, new(big.Int), gas) - contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) + contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr, false)) // We do an AddBalance of zero here, just in order to trigger a touch. // This doesn't matter on Mainnet, where all empties are gone at the time of Byzantium, @@ -553,7 +553,7 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, if err == nil && !maxCodeSizeExceeded { createDataGas := uint64(len(ret)) * params.CreateDataGas if contract.UseGas(createDataGas) { - evm.StateDB.SetCode(address, ret) + evm.StateDB.SetCode(address, ret, false) } else { err = ErrCodeStoreOutOfGas } diff --git a/core/vm/gas_table_test.go b/core/vm/gas_table_test.go index 08e265c744..65973e832c 100644 --- a/core/vm/gas_table_test.go +++ b/core/vm/gas_table_test.go @@ -85,7 +85,7 @@ func TestEIP2200(t *testing.T) { statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) statedb.CreateAccount(address) - statedb.SetCode(address, hexutil.MustDecode(tt.input)) + statedb.SetCode(address, hexutil.MustDecode(tt.input), false) statedb.SetState(address, common.Hash{}, common.BytesToHash([]byte{tt.original})) statedb.Finalise(true) // Push the state into the "original" slot diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 091ba28ff6..73f826a63f 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -488,7 +488,7 @@ func opExtCodeSize(pc *uint64, interpreter *EVMInterpreter, contract *Contract, slot.SetUint64(0) return nil, nil } - slot.SetUint64(uint64(interpreter.evm.StateDB.GetCodeSize(common.BigToAddress(slot)))) + slot.SetUint64(uint64(interpreter.evm.StateDB.GetCodeSize(common.BigToAddress(slot), false))) return nil, nil } @@ -528,7 +528,7 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, contract *Contract, // for EOAs that are not validators, statedb returns nil code = nil } else { - code = interpreter.evm.StateDB.GetCode(addr) + code = interpreter.evm.StateDB.GetCode(addr, false) } codeCopy := getDataBig(code, codeOffset, length) memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy) diff --git a/core/vm/interface.go b/core/vm/interface.go index 3e3994eda9..3b481fd69e 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -38,9 +38,9 @@ type StateDB interface { SetNonce(common.Address, uint64) GetCodeHash(common.Address) common.Hash - GetCode(common.Address) []byte - SetCode(common.Address, []byte) - GetCodeSize(common.Address) int + GetCode(common.Address, bool) []byte + SetCode(common.Address, []byte, bool) + GetCodeSize(common.Address, bool) int ValidatorWrapper(common.Address, bool, bool) (*staking.ValidatorWrapper, error) UpdateValidatorWrapper(common.Address, *staking.ValidatorWrapper) error diff --git a/core/vm/logger_test.go b/core/vm/logger_test.go index 01db3293b7..191733eed2 100644 --- a/core/vm/logger_test.go +++ b/core/vm/logger_test.go @@ -28,10 +28,10 @@ type dummyContractRef struct { calledForEach bool } -func (dummyContractRef) ReturnGas(*big.Int) {} -func (dummyContractRef) Address() common.Address { return common.Address{} } -func (dummyContractRef) Value() *big.Int { return new(big.Int) } -func (dummyContractRef) SetCode(common.Hash, []byte) {} +func (dummyContractRef) ReturnGas(*big.Int) {} +func (dummyContractRef) Address() common.Address { return common.Address{} } +func (dummyContractRef) Value() *big.Int { return new(big.Int) } +func (dummyContractRef) SetCode(common.Hash, []byte, bool) {} func (d *dummyContractRef) ForEachStorage(callback func(key, value common.Hash) bool) { d.calledForEach = true } diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index 5595866daa..d60883a970 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -112,7 +112,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.DB, error) { ) cfg.State.CreateAccount(address) // set the receiver's (the executing contract) code for execution. - cfg.State.SetCode(address, code) + cfg.State.SetCode(address, code, false) // Call the code with the given configuration. ret, _, err := vmenv.Call( sender, diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index a4f86ed8b6..1ad8d19767 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -105,7 +105,7 @@ func TestCall(t *testing.T) { byte(vm.PUSH1), 32, byte(vm.PUSH1), 0, byte(vm.RETURN), - }) + }, false) ret, _, err := Call(address, nil, &Config{State: state}) if err != nil { @@ -158,7 +158,7 @@ func benchmarkEVMCreate(bench *testing.B, code string) { ) statedb.CreateAccount(sender) - statedb.SetCode(receiver, common.FromHex(code)) + statedb.SetCode(receiver, common.FromHex(code), false) runtimeConfig := Config{ Origin: sender, State: statedb, diff --git a/hmy/tracers/block_tracer.go b/hmy/tracers/block_tracer.go index 12391c6fb9..daaf4171d0 100644 --- a/hmy/tracers/block_tracer.go +++ b/hmy/tracers/block_tracer.go @@ -353,7 +353,7 @@ func (jst *ParityBlockTracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, ret := stackPeek(0) if ret.Sign() != 0 { call.to = common.BigToAddress(ret) - call.output = env.StateDB.GetCode(call.to) + call.output = env.StateDB.GetCode(call.to, false) } else if call.err == nil { call.err = errors.New("internal failure") } diff --git a/hmy/tracers/tracer.go b/hmy/tracers/tracer.go index bc349a5147..69d52e3fdc 100644 --- a/hmy/tracers/tracer.go +++ b/hmy/tracers/tracer.go @@ -210,7 +210,7 @@ func (dw *dbWrapper) pushObject(vm *duktape.Context) { // Push the wrapper for statedb.GetCode vm.PushGoFunction(func(ctx *duktape.Context) int { - code := dw.db.GetCode(common.BytesToAddress(popSlice(ctx))) + code := dw.db.GetCode(common.BytesToAddress(popSlice(ctx)), false) ptr := ctx.PushFixedBuffer(len(code)) copy(makeSlice(ptr, uint(len(code))), code) diff --git a/rosetta/services/block.go b/rosetta/services/block.go index c488c5ff95..8a939b1081 100644 --- a/rosetta/services/block.go +++ b/rosetta/services/block.go @@ -178,11 +178,11 @@ func (s *BlockAPI) BlockTransaction( // check for contract related operations, if it is a plain transaction. if txInfo.tx.To() != nil { // possible call to existing contract so fetch relevant data - contractInfo.ContractCode = state.GetCode(*txInfo.tx.To()) + contractInfo.ContractCode = state.GetCode(*txInfo.tx.To(), false) contractInfo.ContractAddress = txInfo.tx.To() } else { // contract creation, so address is in receipt - contractInfo.ContractCode = state.GetCode(txInfo.receipt.ContractAddress) + contractInfo.ContractCode = state.GetCode(txInfo.receipt.ContractAddress, false) contractInfo.ContractAddress = &txInfo.receipt.ContractAddress } contractInfo.ExecutionResult, rosettaError = s.getTransactionTrace(ctx, blk, txInfo) diff --git a/rosetta/services/construction_check.go b/rosetta/services/construction_check.go index c842770ab7..d08ed6e8f3 100644 --- a/rosetta/services/construction_check.go +++ b/rosetta/services/construction_check.go @@ -289,7 +289,7 @@ func (s *ConstructAPI) ConstructionMetadata( GasPrice: sugNativePrice, GasLimit: estGasUsed, Transaction: options.TransactionMetadata, - ContractCode: state.GetCode(contractAddress), + ContractCode: state.GetCode(contractAddress, false), EvmErrorMessage: evmErrorMsg, EvmReturn: evmReturn, }) diff --git a/rpc/contract.go b/rpc/contract.go index abcb4f9418..daed35edd7 100644 --- a/rpc/contract.go +++ b/rpc/contract.go @@ -123,7 +123,7 @@ func (s *PublicContractService) GetCode( DoMetricRPCQueryInfo(GetCode, FailedNumber) return nil, err } - code := state.GetCode(address) + code := state.GetCode(address, false) // Response output is the same for all versions return code, state.Error() diff --git a/staking/availability/measure_test.go b/staking/availability/measure_test.go index 8f0f502b1e..9a35ad80eb 100644 --- a/staking/availability/measure_test.go +++ b/staking/availability/measure_test.go @@ -434,8 +434,8 @@ func (ctx *incStateTestCtx) checkAddrIncStateByType(addr common.Address, typeInc // checkHmyNodeStateChangeByAddr checks the state change for hmy nodes. Since hmy nodes does not // have wrapper, it is supposed to be unchanged in code field func (ctx *incStateTestCtx) checkHmyNodeStateChangeByAddr(addr common.Address) error { - snapCode := ctx.snapState.GetCode(addr) - curCode := ctx.state.GetCode(addr) + snapCode := ctx.snapState.GetCode(addr, false) + curCode := ctx.state.GetCode(addr, false) if !reflect.DeepEqual(snapCode, curCode) { return errors.New("code not expected") } @@ -618,7 +618,7 @@ func (state testStateDB) UpdateValidatorWrapper(addr common.Address, wrapper *st return nil } -func (state testStateDB) GetCode(addr common.Address) []byte { +func (state testStateDB) GetCode(addr common.Address, isValidatorCode bool) []byte { wrapper, ok := state[addr] if !ok { return nil From 49e984e268d4672d952da6f73019be734aaa0624 Mon Sep 17 00:00:00 2001 From: Gheis Mohammadi Date: Tue, 25 Apr 2023 01:30:41 +0800 Subject: [PATCH 391/420] add snapshot feature to state db (#4406) --- core/state/snapshot/account.go | 87 ++ core/state/snapshot/context.go | 241 ++++ core/state/snapshot/conversion.go | 383 ++++++ core/state/snapshot/difflayer.go | 559 +++++++++ core/state/snapshot/difflayer_test.go | 399 +++++++ core/state/snapshot/disklayer.go | 166 +++ core/state/snapshot/disklayer_test.go | 574 +++++++++ core/state/snapshot/generate.go | 756 ++++++++++++ core/state/snapshot/generate_test.go | 861 ++++++++++++++ core/state/snapshot/holdable_iterator.go | 97 ++ core/state/snapshot/holdable_iterator_test.go | 163 +++ core/state/snapshot/iterator.go | 400 +++++++ core/state/snapshot/iterator_binary.go | 213 ++++ core/state/snapshot/iterator_fast.go | 350 ++++++ core/state/snapshot/iterator_test.go | 1047 +++++++++++++++++ core/state/snapshot/journal.go | 374 ++++++ core/state/snapshot/metrics.go | 53 + core/state/snapshot/snapshot.go | 854 ++++++++++++++ core/state/snapshot/snapshot_test.go | 488 ++++++++ core/state/snapshot/sort.go | 36 + core/state/snapshot/utils.go | 165 +++ 21 files changed, 8266 insertions(+) create mode 100644 core/state/snapshot/account.go create mode 100644 core/state/snapshot/context.go create mode 100644 core/state/snapshot/conversion.go create mode 100644 core/state/snapshot/difflayer.go create mode 100644 core/state/snapshot/difflayer_test.go create mode 100644 core/state/snapshot/disklayer.go create mode 100644 core/state/snapshot/disklayer_test.go create mode 100644 core/state/snapshot/generate.go create mode 100644 core/state/snapshot/generate_test.go create mode 100644 core/state/snapshot/holdable_iterator.go create mode 100644 core/state/snapshot/holdable_iterator_test.go create mode 100644 core/state/snapshot/iterator.go create mode 100644 core/state/snapshot/iterator_binary.go create mode 100644 core/state/snapshot/iterator_fast.go create mode 100644 core/state/snapshot/iterator_test.go create mode 100644 core/state/snapshot/journal.go create mode 100644 core/state/snapshot/metrics.go create mode 100644 core/state/snapshot/snapshot.go create mode 100644 core/state/snapshot/snapshot_test.go create mode 100644 core/state/snapshot/sort.go create mode 100644 core/state/snapshot/utils.go diff --git a/core/state/snapshot/account.go b/core/state/snapshot/account.go new file mode 100644 index 0000000000..b5634972ad --- /dev/null +++ b/core/state/snapshot/account.go @@ -0,0 +1,87 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "bytes" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// Account is a modified version of a state.Account, where the root is replaced +// with a byte slice. This format can be used to represent full-consensus format +// or slim-snapshot format which replaces the empty root and code hash as nil +// byte slice. +type Account struct { + Nonce uint64 + Balance *big.Int + Root []byte + CodeHash []byte +} + +// SlimAccount converts a state.Account content into a slim snapshot account +func SlimAccount(nonce uint64, balance *big.Int, root common.Hash, codehash []byte) Account { + slim := Account{ + Nonce: nonce, + Balance: balance, + } + if root != types.EmptyRootHash { + slim.Root = root[:] + } + if !bytes.Equal(codehash, types.EmptyCodeHash[:]) { + slim.CodeHash = codehash + } + return slim +} + +// SlimAccountRLP converts a state.Account content into a slim snapshot +// version RLP encoded. +func SlimAccountRLP(nonce uint64, balance *big.Int, root common.Hash, codehash []byte) []byte { + data, err := rlp.EncodeToBytes(SlimAccount(nonce, balance, root, codehash)) + if err != nil { + panic(err) + } + return data +} + +// FullAccount decodes the data on the 'slim RLP' format and return +// the consensus format account. +func FullAccount(data []byte) (Account, error) { + var account Account + if err := rlp.DecodeBytes(data, &account); err != nil { + return Account{}, err + } + if len(account.Root) == 0 { + account.Root = types.EmptyRootHash[:] + } + if len(account.CodeHash) == 0 { + account.CodeHash = types.EmptyCodeHash[:] + } + return account, nil +} + +// FullAccountRLP converts data on the 'slim RLP' format into the full RLP-format. +func FullAccountRLP(data []byte) ([]byte, error) { + account, err := FullAccount(data) + if err != nil { + return nil, err + } + return rlp.EncodeToBytes(account) +} diff --git a/core/state/snapshot/context.go b/core/state/snapshot/context.go new file mode 100644 index 0000000000..16a52c81f5 --- /dev/null +++ b/core/state/snapshot/context.go @@ -0,0 +1,241 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "bytes" + "encoding/binary" + "errors" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/harmony-one/harmony/core/rawdb" + "github.com/harmony-one/harmony/internal/utils" +) + +const ( + snapAccount = "account" // Identifier of account snapshot generation + snapStorage = "storage" // Identifier of storage snapshot generation +) + +// generatorStats is a collection of statistics gathered by the snapshot generator +// for logging purposes. +type generatorStats struct { + origin uint64 // Origin prefix where generation started + start time.Time // Timestamp when generation started + accounts uint64 // Number of accounts indexed(generated or recovered) + slots uint64 // Number of storage slots indexed(generated or recovered) + dangling uint64 // Number of dangling storage slots + storage common.StorageSize // Total account and storage slot size(generation or recovery) +} + +// Log creates an contextual log with the given message and the context pulled +// from the internally maintained statistics. +func (gs *generatorStats) Log(msg string, root common.Hash, marker []byte) { + var ctx []interface{} + if root != (common.Hash{}) { + ctx = append(ctx, []interface{}{"root", root}...) + } + // Figure out whether we're after or within an account + switch len(marker) { + case common.HashLength: + ctx = append(ctx, []interface{}{"at", common.BytesToHash(marker)}...) + case 2 * common.HashLength: + ctx = append(ctx, []interface{}{ + "in", common.BytesToHash(marker[:common.HashLength]), + "at", common.BytesToHash(marker[common.HashLength:]), + }...) + } + // Add the usual measurements + ctx = append(ctx, []interface{}{ + "accounts", gs.accounts, + "slots", gs.slots, + "storage", gs.storage, + "dangling", gs.dangling, + "elapsed", common.PrettyDuration(time.Since(gs.start)), + }...) + // Calculate the estimated indexing time based on current stats + if len(marker) > 0 { + if done := binary.BigEndian.Uint64(marker[:8]) - gs.origin; done > 0 { + left := math.MaxUint64 - binary.BigEndian.Uint64(marker[:8]) + + speed := done/uint64(time.Since(gs.start)/time.Millisecond+1) + 1 // +1s to avoid division by zero + ctx = append(ctx, []interface{}{ + "eta", common.PrettyDuration(time.Duration(left/speed) * time.Millisecond), + }...) + } + } + utils.Logger().Info().Msg(msg) +} + +// generatorContext carries a few global values to be shared by all generation functions. +type generatorContext struct { + stats *generatorStats // Generation statistic collection + db ethdb.KeyValueStore // Key-value store containing the snapshot data + account *holdableIterator // Iterator of account snapshot data + storage *holdableIterator // Iterator of storage snapshot data + batch ethdb.Batch // Database batch for writing batch data atomically + logged time.Time // The timestamp when last generation progress was displayed +} + +// newGeneratorContext initializes the context for generation. +func newGeneratorContext(stats *generatorStats, db ethdb.KeyValueStore, accMarker []byte, storageMarker []byte) *generatorContext { + ctx := &generatorContext{ + stats: stats, + db: db, + batch: db.NewBatch(), + logged: time.Now(), + } + ctx.openIterator(snapAccount, accMarker) + ctx.openIterator(snapStorage, storageMarker) + return ctx +} + +// openIterator constructs global account and storage snapshot iterators +// at the interrupted position. These iterators should be reopened from time +// to time to avoid blocking leveldb compaction for a long time. +func (ctx *generatorContext) openIterator(kind string, start []byte) { + if kind == snapAccount { + iter := ctx.db.NewIterator(rawdb.SnapshotAccountPrefix, start) + ctx.account = newHoldableIterator(rawdb.NewKeyLengthIterator(iter, 1+common.HashLength)) + return + } + iter := ctx.db.NewIterator(rawdb.SnapshotStoragePrefix, start) + ctx.storage = newHoldableIterator(rawdb.NewKeyLengthIterator(iter, 1+2*common.HashLength)) +} + +// reopenIterator releases the specified snapshot iterator and re-open it +// in the next position. It's aimed for not blocking leveldb compaction. +func (ctx *generatorContext) reopenIterator(kind string) { + // Shift iterator one more step, so that we can reopen + // the iterator at the right position. + var iter = ctx.account + if kind == snapStorage { + iter = ctx.storage + } + hasNext := iter.Next() + if !hasNext { + // Iterator exhausted, release forever and create an already exhausted virtual iterator + iter.Release() + if kind == snapAccount { + ctx.account = newHoldableIterator(memorydb.New().NewIterator(nil, nil)) + return + } + ctx.storage = newHoldableIterator(memorydb.New().NewIterator(nil, nil)) + return + } + next := iter.Key() + iter.Release() + ctx.openIterator(kind, next[1:]) +} + +// close releases all the held resources. +func (ctx *generatorContext) close() { + ctx.account.Release() + ctx.storage.Release() +} + +// iterator returns the corresponding iterator specified by the kind. +func (ctx *generatorContext) iterator(kind string) *holdableIterator { + if kind == snapAccount { + return ctx.account + } + return ctx.storage +} + +// removeStorageBefore deletes all storage entries which are located before +// the specified account. When the iterator touches the storage entry which +// is located in or outside the given account, it stops and holds the current +// iterated element locally. +func (ctx *generatorContext) removeStorageBefore(account common.Hash) { + var ( + count uint64 + start = time.Now() + iter = ctx.storage + ) + for iter.Next() { + key := iter.Key() + if bytes.Compare(key[1:1+common.HashLength], account.Bytes()) >= 0 { + iter.Hold() + break + } + count++ + ctx.batch.Delete(key) + if ctx.batch.ValueSize() > ethdb.IdealBatchSize { + ctx.batch.Write() + ctx.batch.Reset() + } + } + ctx.stats.dangling += count + snapStorageCleanCounter.Inc(time.Since(start).Nanoseconds()) +} + +// removeStorageAt deletes all storage entries which are located in the specified +// account. When the iterator touches the storage entry which is outside the given +// account, it stops and holds the current iterated element locally. An error will +// be returned if the initial position of iterator is not in the given account. +func (ctx *generatorContext) removeStorageAt(account common.Hash) error { + var ( + count int64 + start = time.Now() + iter = ctx.storage + ) + for iter.Next() { + key := iter.Key() + cmp := bytes.Compare(key[1:1+common.HashLength], account.Bytes()) + if cmp < 0 { + return errors.New("invalid iterator position") + } + if cmp > 0 { + iter.Hold() + break + } + count++ + ctx.batch.Delete(key) + if ctx.batch.ValueSize() > ethdb.IdealBatchSize { + ctx.batch.Write() + ctx.batch.Reset() + } + } + snapWipedStorageMeter.Mark(count) + snapStorageCleanCounter.Inc(time.Since(start).Nanoseconds()) + return nil +} + +// removeStorageLeft deletes all storage entries which are located after +// the current iterator position. +func (ctx *generatorContext) removeStorageLeft() { + var ( + count uint64 + start = time.Now() + iter = ctx.storage + ) + for iter.Next() { + count++ + ctx.batch.Delete(iter.Key()) + if ctx.batch.ValueSize() > ethdb.IdealBatchSize { + ctx.batch.Write() + ctx.batch.Reset() + } + } + ctx.stats.dangling += count + snapDanglingStorageMeter.Mark(int64(count)) + snapStorageCleanCounter.Inc(time.Since(start).Nanoseconds()) +} diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go new file mode 100644 index 0000000000..fd42167376 --- /dev/null +++ b/core/state/snapshot/conversion.go @@ -0,0 +1,383 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "math" + "runtime" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/harmony-one/harmony/core/rawdb" + "github.com/harmony-one/harmony/internal/utils" +) + +// trieKV represents a trie key-value pair +type trieKV struct { + key common.Hash + value []byte +} + +type ( + // trieGeneratorFn is the interface of trie generation which can + // be implemented by different trie algorithm. + trieGeneratorFn func(db ethdb.KeyValueWriter, scheme string, owner common.Hash, in chan (trieKV), out chan (common.Hash)) + + // leafCallbackFn is the callback invoked at the leaves of the trie, + // returns the subtrie root with the specified subtrie identifier. + leafCallbackFn func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) +) + +// GenerateAccountTrieRoot takes an account iterator and reproduces the root hash. +func GenerateAccountTrieRoot(it AccountIterator) (common.Hash, error) { + return generateTrieRoot(nil, "", it, common.Hash{}, stackTrieGenerate, nil, newGenerateStats(), true) +} + +// GenerateStorageTrieRoot takes a storage iterator and reproduces the root hash. +func GenerateStorageTrieRoot(account common.Hash, it StorageIterator) (common.Hash, error) { + return generateTrieRoot(nil, "", it, account, stackTrieGenerate, nil, newGenerateStats(), true) +} + +// GenerateTrie takes the whole snapshot tree as the input, traverses all the +// accounts as well as the corresponding storages and regenerate the whole state +// (account trie + all storage tries). +func GenerateTrie(snaptree *Tree, root common.Hash, src ethdb.Database, dst ethdb.KeyValueWriter) error { + // Traverse all state by snapshot, re-generate the whole state trie + acctIt, err := snaptree.AccountIterator(root, common.Hash{}) + if err != nil { + return err // The required snapshot might not exist. + } + defer acctIt.Release() + + scheme := snaptree.triedb.Scheme() + got, err := generateTrieRoot(dst, scheme, acctIt, common.Hash{}, stackTrieGenerate, func(dst ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) { + // Migrate the code first, commit the contract code into the tmp db. + if codeHash != types.EmptyCodeHash { + code := rawdb.ReadCode(src, codeHash) + if len(code) == 0 { + return common.Hash{}, errors.New("failed to read code") + } + rawdb.WriteCode(dst, codeHash, code) + } + // Then migrate all storage trie nodes into the tmp db. + storageIt, err := snaptree.StorageIterator(root, accountHash, common.Hash{}) + if err != nil { + return common.Hash{}, err + } + defer storageIt.Release() + + hash, err := generateTrieRoot(dst, scheme, storageIt, accountHash, stackTrieGenerate, nil, stat, false) + if err != nil { + return common.Hash{}, err + } + return hash, nil + }, newGenerateStats(), true) + + if err != nil { + return err + } + if got != root { + return fmt.Errorf("state root hash mismatch: got %x, want %x", got, root) + } + return nil +} + +// generateStats is a collection of statistics gathered by the trie generator +// for logging purposes. +type generateStats struct { + head common.Hash + start time.Time + + accounts uint64 // Number of accounts done (including those being crawled) + slots uint64 // Number of storage slots done (including those being crawled) + + slotsStart map[common.Hash]time.Time // Start time for account slot crawling + slotsHead map[common.Hash]common.Hash // Slot head for accounts being crawled + + lock sync.RWMutex +} + +// newGenerateStats creates a new generator stats. +func newGenerateStats() *generateStats { + return &generateStats{ + slotsStart: make(map[common.Hash]time.Time), + slotsHead: make(map[common.Hash]common.Hash), + start: time.Now(), + } +} + +// progressAccounts updates the generator stats for the account range. +func (stat *generateStats) progressAccounts(account common.Hash, done uint64) { + stat.lock.Lock() + defer stat.lock.Unlock() + + stat.accounts += done + stat.head = account +} + +// finishAccounts updates the generator stats for the finished account range. +func (stat *generateStats) finishAccounts(done uint64) { + stat.lock.Lock() + defer stat.lock.Unlock() + + stat.accounts += done +} + +// progressContract updates the generator stats for a specific in-progress contract. +func (stat *generateStats) progressContract(account common.Hash, slot common.Hash, done uint64) { + stat.lock.Lock() + defer stat.lock.Unlock() + + stat.slots += done + stat.slotsHead[account] = slot + if _, ok := stat.slotsStart[account]; !ok { + stat.slotsStart[account] = time.Now() + } +} + +// finishContract updates the generator stats for a specific just-finished contract. +func (stat *generateStats) finishContract(account common.Hash, done uint64) { + stat.lock.Lock() + defer stat.lock.Unlock() + + stat.slots += done + delete(stat.slotsHead, account) + delete(stat.slotsStart, account) +} + +// report prints the cumulative progress statistic smartly. +func (stat *generateStats) report() { + stat.lock.RLock() + defer stat.lock.RUnlock() + + ctx := []interface{}{ + "accounts", stat.accounts, + "slots", stat.slots, + "elapsed", common.PrettyDuration(time.Since(stat.start)), + } + if stat.accounts > 0 { + // If there's progress on the account trie, estimate the time to finish crawling it + if done := binary.BigEndian.Uint64(stat.head[:8]) / stat.accounts; done > 0 { + var ( + left = (math.MaxUint64 - binary.BigEndian.Uint64(stat.head[:8])) / stat.accounts + speed = done/uint64(time.Since(stat.start)/time.Millisecond+1) + 1 // +1s to avoid division by zero + eta = time.Duration(left/speed) * time.Millisecond + ) + // If there are large contract crawls in progress, estimate their finish time + for acc, head := range stat.slotsHead { + start := stat.slotsStart[acc] + if done := binary.BigEndian.Uint64(head[:8]); done > 0 { + var ( + left = math.MaxUint64 - binary.BigEndian.Uint64(head[:8]) + speed = done/uint64(time.Since(start)/time.Millisecond+1) + 1 // +1s to avoid division by zero + ) + // Override the ETA if larger than the largest until now + if slotETA := time.Duration(left/speed) * time.Millisecond; eta < slotETA { + eta = slotETA + } + } + } + ctx = append(ctx, []interface{}{ + "eta", common.PrettyDuration(eta), + }...) + } + } + utils.Logger().Info().Msg("Iterating state snapshot") +} + +// reportDone prints the last log when the whole generation is finished. +func (stat *generateStats) reportDone() { + stat.lock.RLock() + defer stat.lock.RUnlock() + + var ctx []interface{} + ctx = append(ctx, []interface{}{"accounts", stat.accounts}...) + if stat.slots != 0 { + ctx = append(ctx, []interface{}{"slots", stat.slots}...) + } + ctx = append(ctx, []interface{}{"elapsed", common.PrettyDuration(time.Since(stat.start))}...) + utils.Logger().Info().Msg("Iterated snapshot") +} + +// runReport periodically prints the progress information. +func runReport(stats *generateStats, stop chan bool) { + timer := time.NewTimer(0) + defer timer.Stop() + + for { + select { + case <-timer.C: + stats.report() + timer.Reset(time.Second * 8) + case success := <-stop: + if success { + stats.reportDone() + } + return + } + } +} + +// generateTrieRoot generates the trie hash based on the snapshot iterator. +// It can be used for generating account trie, storage trie or even the +// whole state which connects the accounts and the corresponding storages. +func generateTrieRoot(db ethdb.KeyValueWriter, scheme string, it Iterator, account common.Hash, generatorFn trieGeneratorFn, leafCallback leafCallbackFn, stats *generateStats, report bool) (common.Hash, error) { + var ( + in = make(chan trieKV) // chan to pass leaves + out = make(chan common.Hash, 1) // chan to collect result + stoplog = make(chan bool, 1) // 1-size buffer, works when logging is not enabled + wg sync.WaitGroup + ) + // Spin up a go-routine for trie hash re-generation + wg.Add(1) + go func() { + defer wg.Done() + generatorFn(db, scheme, account, in, out) + }() + // Spin up a go-routine for progress logging + if report && stats != nil { + wg.Add(1) + go func() { + defer wg.Done() + runReport(stats, stoplog) + }() + } + // Create a semaphore to assign tasks and collect results through. We'll pre- + // fill it with nils, thus using the same channel for both limiting concurrent + // processing and gathering results. + threads := runtime.NumCPU() + results := make(chan error, threads) + for i := 0; i < threads; i++ { + results <- nil // fill the semaphore + } + // stop is a helper function to shutdown the background threads + // and return the re-generated trie hash. + stop := func(fail error) (common.Hash, error) { + close(in) + result := <-out + for i := 0; i < threads; i++ { + if err := <-results; err != nil && fail == nil { + fail = err + } + } + stoplog <- fail == nil + + wg.Wait() + return result, fail + } + var ( + logged = time.Now() + processed = uint64(0) + leaf trieKV + ) + // Start to feed leaves + for it.Next() { + if account == (common.Hash{}) { + var ( + err error + fullData []byte + ) + if leafCallback == nil { + fullData, err = FullAccountRLP(it.(AccountIterator).Account()) + if err != nil { + return stop(err) + } + } else { + // Wait until the semaphore allows us to continue, aborting if + // a sub-task failed + if err := <-results; err != nil { + results <- nil // stop will drain the results, add a noop back for this error we just consumed + return stop(err) + } + // Fetch the next account and process it concurrently + account, err := FullAccount(it.(AccountIterator).Account()) + if err != nil { + return stop(err) + } + go func(hash common.Hash) { + subroot, err := leafCallback(db, hash, common.BytesToHash(account.CodeHash), stats) + if err != nil { + results <- err + return + } + if !bytes.Equal(account.Root, subroot.Bytes()) { + results <- fmt.Errorf("invalid subroot(path %x), want %x, have %x", hash, account.Root, subroot) + return + } + results <- nil + }(it.Hash()) + fullData, err = rlp.EncodeToBytes(account) + if err != nil { + return stop(err) + } + } + leaf = trieKV{it.Hash(), fullData} + } else { + leaf = trieKV{it.Hash(), common.CopyBytes(it.(StorageIterator).Slot())} + } + in <- leaf + + // Accumulate the generation statistic if it's required. + processed++ + if time.Since(logged) > 3*time.Second && stats != nil { + if account == (common.Hash{}) { + stats.progressAccounts(it.Hash(), processed) + } else { + stats.progressContract(account, it.Hash(), processed) + } + logged, processed = time.Now(), 0 + } + } + // Commit the last part statistic. + if processed > 0 && stats != nil { + if account == (common.Hash{}) { + stats.finishAccounts(processed) + } else { + stats.finishContract(account, processed) + } + } + return stop(nil) +} + +func stackTrieGenerate(db ethdb.KeyValueWriter, scheme string, owner common.Hash, in chan trieKV, out chan common.Hash) { + var nodeWriter trie.NodeWriteFunc + if db != nil { + nodeWriter = func(owner common.Hash, path []byte, hash common.Hash, blob []byte) { + rawdb.WriteTrieNode(db, owner, path, hash, blob, scheme) + } + } + t := trie.NewStackTrieWithOwner(nodeWriter, owner) + for leaf := range in { + t.TryUpdate(leaf.key[:], leaf.value) + } + var root common.Hash + if db == nil { + root = t.Hash() + } else { + root, _ = t.Commit() + } + out <- root +} diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go new file mode 100644 index 0000000000..f916a020e7 --- /dev/null +++ b/core/state/snapshot/difflayer.go @@ -0,0 +1,559 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "encoding/binary" + "fmt" + "math" + "math/rand" + "sort" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" + bloomfilter "github.com/holiman/bloomfilter/v2" +) + +var ( + // aggregatorMemoryLimit is the maximum size of the bottom-most diff layer + // that aggregates the writes from above until it's flushed into the disk + // layer. + // + // Note, bumping this up might drastically increase the size of the bloom + // filters that's stored in every diff layer. Don't do that without fully + // understanding all the implications. + aggregatorMemoryLimit = uint64(4 * 1024 * 1024) + + // aggregatorItemLimit is an approximate number of items that will end up + // in the agregator layer before it's flushed out to disk. A plain account + // weighs around 14B (+hash), a storage slot 32B (+hash), a deleted slot + // 0B (+hash). Slots are mostly set/unset in lockstep, so that average at + // 16B (+hash). All in all, the average entry seems to be 15+32=47B. Use a + // smaller number to be on the safe side. + aggregatorItemLimit = aggregatorMemoryLimit / 42 + + // bloomTargetError is the target false positive rate when the aggregator + // layer is at its fullest. The actual value will probably move around up + // and down from this number, it's mostly a ballpark figure. + // + // Note, dropping this down might drastically increase the size of the bloom + // filters that's stored in every diff layer. Don't do that without fully + // understanding all the implications. + bloomTargetError = 0.02 + + // bloomSize is the ideal bloom filter size given the maximum number of items + // it's expected to hold and the target false positive error rate. + bloomSize = math.Ceil(float64(aggregatorItemLimit) * math.Log(bloomTargetError) / math.Log(1/math.Pow(2, math.Log(2)))) + + // bloomFuncs is the ideal number of bits a single entry should set in the + // bloom filter to keep its size to a minimum (given it's size and maximum + // entry count). + bloomFuncs = math.Round((bloomSize / float64(aggregatorItemLimit)) * math.Log(2)) + + // the bloom offsets are runtime constants which determines which part of the + // account/storage hash the hasher functions looks at, to determine the + // bloom key for an account/slot. This is randomized at init(), so that the + // global population of nodes do not all display the exact same behaviour with + // regards to bloom content + bloomDestructHasherOffset = 0 + bloomAccountHasherOffset = 0 + bloomStorageHasherOffset = 0 +) + +func init() { + // Init the bloom offsets in the range [0:24] (requires 8 bytes) + bloomDestructHasherOffset = rand.Intn(25) + bloomAccountHasherOffset = rand.Intn(25) + bloomStorageHasherOffset = rand.Intn(25) + + // The destruct and account blooms must be different, as the storage slots + // will check for destruction too for every bloom miss. It should not collide + // with modified accounts. + for bloomAccountHasherOffset == bloomDestructHasherOffset { + bloomAccountHasherOffset = rand.Intn(25) + } +} + +// diffLayer represents a collection of modifications made to a state snapshot +// after running a block on top. It contains one sorted list for the account trie +// and one-one list for each storage tries. +// +// The goal of a diff layer is to act as a journal, tracking recent modifications +// made to the state, that have not yet graduated into a semi-immutable state. +type diffLayer struct { + origin *diskLayer // Base disk layer to directly use on bloom misses + parent snapshot // Parent snapshot modified by this one, never nil + memory uint64 // Approximate guess as to how much memory we use + + root common.Hash // Root hash to which this snapshot diff belongs to + stale uint32 // Signals that the layer became stale (state progressed) + + // destructSet is a very special helper marker. If an account is marked as + // deleted, then it's recorded in this set. However it's allowed that an account + // is included here but still available in other sets(e.g. storageData). The + // reason is the diff layer includes all the changes in a *block*. It can + // happen that in the tx_1, account A is self-destructed while in the tx_2 + // it's recreated. But we still need this marker to indicate the "old" A is + // deleted, all data in other set belongs to the "new" A. + destructSet map[common.Hash]struct{} // Keyed markers for deleted (and potentially) recreated accounts + accountList []common.Hash // List of account for iteration. If it exists, it's sorted, otherwise it's nil + accountData map[common.Hash][]byte // Keyed accounts for direct retrieval (nil means deleted) + storageList map[common.Hash][]common.Hash // List of storage slots for iterated retrievals, one per account. Any existing lists are sorted if non-nil + storageData map[common.Hash]map[common.Hash][]byte // Keyed storage slots for direct retrieval. one per account (nil means deleted) + + diffed *bloomfilter.Filter // Bloom filter tracking all the diffed items up to the disk layer + + lock sync.RWMutex +} + +// destructBloomHasher is a wrapper around a common.Hash to satisfy the interface +// API requirements of the bloom library used. It's used to convert a destruct +// event into a 64 bit mini hash. +type destructBloomHasher common.Hash + +func (h destructBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") } +func (h destructBloomHasher) Sum(b []byte) []byte { panic("not implemented") } +func (h destructBloomHasher) Reset() { panic("not implemented") } +func (h destructBloomHasher) BlockSize() int { panic("not implemented") } +func (h destructBloomHasher) Size() int { return 8 } +func (h destructBloomHasher) Sum64() uint64 { + return binary.BigEndian.Uint64(h[bloomDestructHasherOffset : bloomDestructHasherOffset+8]) +} + +// accountBloomHasher is a wrapper around a common.Hash to satisfy the interface +// API requirements of the bloom library used. It's used to convert an account +// hash into a 64 bit mini hash. +type accountBloomHasher common.Hash + +func (h accountBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") } +func (h accountBloomHasher) Sum(b []byte) []byte { panic("not implemented") } +func (h accountBloomHasher) Reset() { panic("not implemented") } +func (h accountBloomHasher) BlockSize() int { panic("not implemented") } +func (h accountBloomHasher) Size() int { return 8 } +func (h accountBloomHasher) Sum64() uint64 { + return binary.BigEndian.Uint64(h[bloomAccountHasherOffset : bloomAccountHasherOffset+8]) +} + +// storageBloomHasher is a wrapper around a [2]common.Hash to satisfy the interface +// API requirements of the bloom library used. It's used to convert an account +// hash into a 64 bit mini hash. +type storageBloomHasher [2]common.Hash + +func (h storageBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") } +func (h storageBloomHasher) Sum(b []byte) []byte { panic("not implemented") } +func (h storageBloomHasher) Reset() { panic("not implemented") } +func (h storageBloomHasher) BlockSize() int { panic("not implemented") } +func (h storageBloomHasher) Size() int { return 8 } +func (h storageBloomHasher) Sum64() uint64 { + return binary.BigEndian.Uint64(h[0][bloomStorageHasherOffset:bloomStorageHasherOffset+8]) ^ + binary.BigEndian.Uint64(h[1][bloomStorageHasherOffset:bloomStorageHasherOffset+8]) +} + +// newDiffLayer creates a new diff on top of an existing snapshot, whether that's a low +// level persistent database or a hierarchical diff already. +func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { + // Create the new layer with some pre-allocated data segments + dl := &diffLayer{ + parent: parent, + root: root, + destructSet: destructs, + accountData: accounts, + storageData: storage, + storageList: make(map[common.Hash][]common.Hash), + } + switch parent := parent.(type) { + case *diskLayer: + dl.rebloom(parent) + case *diffLayer: + dl.rebloom(parent.origin) + default: + panic("unknown parent type") + } + // Sanity check that accounts or storage slots are never nil + for accountHash, blob := range accounts { + if blob == nil { + panic(fmt.Sprintf("account %#x nil", accountHash)) + } + // Determine memory size and track the dirty writes + dl.memory += uint64(common.HashLength + len(blob)) + snapshotDirtyAccountWriteMeter.Mark(int64(len(blob))) + } + for accountHash, slots := range storage { + if slots == nil { + panic(fmt.Sprintf("storage %#x nil", accountHash)) + } + // Determine memory size and track the dirty writes + for _, data := range slots { + dl.memory += uint64(common.HashLength + len(data)) + snapshotDirtyStorageWriteMeter.Mark(int64(len(data))) + } + } + dl.memory += uint64(len(destructs) * common.HashLength) + return dl +} + +// rebloom discards the layer's current bloom and rebuilds it from scratch based +// on the parent's and the local diffs. +func (dl *diffLayer) rebloom(origin *diskLayer) { + dl.lock.Lock() + defer dl.lock.Unlock() + + defer func(start time.Time) { + snapshotBloomIndexTimer.Update(time.Since(start)) + }(time.Now()) + + // Inject the new origin that triggered the rebloom + dl.origin = origin + + // Retrieve the parent bloom or create a fresh empty one + if parent, ok := dl.parent.(*diffLayer); ok { + parent.lock.RLock() + dl.diffed, _ = parent.diffed.Copy() + parent.lock.RUnlock() + } else { + dl.diffed, _ = bloomfilter.New(uint64(bloomSize), uint64(bloomFuncs)) + } + // Iterate over all the accounts and storage slots and index them + for hash := range dl.destructSet { + dl.diffed.Add(destructBloomHasher(hash)) + } + for hash := range dl.accountData { + dl.diffed.Add(accountBloomHasher(hash)) + } + for accountHash, slots := range dl.storageData { + for storageHash := range slots { + dl.diffed.Add(storageBloomHasher{accountHash, storageHash}) + } + } + // Calculate the current false positive rate and update the error rate meter. + // This is a bit cheating because subsequent layers will overwrite it, but it + // should be fine, we're only interested in ballpark figures. + k := float64(dl.diffed.K()) + n := float64(dl.diffed.N()) + m := float64(dl.diffed.M()) + snapshotBloomErrorGauge.Update(math.Pow(1.0-math.Exp((-k)*(n+0.5)/(m-1)), k)) +} + +// Root returns the root hash for which this snapshot was made. +func (dl *diffLayer) Root() common.Hash { + return dl.root +} + +// Parent returns the subsequent layer of a diff layer. +func (dl *diffLayer) Parent() snapshot { + dl.lock.RLock() + defer dl.lock.RUnlock() + + return dl.parent +} + +// Stale return whether this layer has become stale (was flattened across) or if +// it's still live. +func (dl *diffLayer) Stale() bool { + return atomic.LoadUint32(&dl.stale) != 0 +} + +// Account directly retrieves the account associated with a particular hash in +// the snapshot slim data format. +func (dl *diffLayer) Account(hash common.Hash) (*Account, error) { + data, err := dl.AccountRLP(hash) + if err != nil { + return nil, err + } + if len(data) == 0 { // can be both nil and []byte{} + return nil, nil + } + account := new(Account) + if err := rlp.DecodeBytes(data, account); err != nil { + panic(err) + } + return account, nil +} + +// AccountRLP directly retrieves the account RLP associated with a particular +// hash in the snapshot slim data format. +// +// Note the returned account is not a copy, please don't modify it. +func (dl *diffLayer) AccountRLP(hash common.Hash) ([]byte, error) { + // Check the bloom filter first whether there's even a point in reaching into + // all the maps in all the layers below + dl.lock.RLock() + hit := dl.diffed.Contains(accountBloomHasher(hash)) + if !hit { + hit = dl.diffed.Contains(destructBloomHasher(hash)) + } + var origin *diskLayer + if !hit { + origin = dl.origin // extract origin while holding the lock + } + dl.lock.RUnlock() + + // If the bloom filter misses, don't even bother with traversing the memory + // diff layers, reach straight into the bottom persistent disk layer + if origin != nil { + snapshotBloomAccountMissMeter.Mark(1) + return origin.AccountRLP(hash) + } + // The bloom filter hit, start poking in the internal maps + return dl.accountRLP(hash, 0) +} + +// accountRLP is an internal version of AccountRLP that skips the bloom filter +// checks and uses the internal maps to try and retrieve the data. It's meant +// to be used if a higher layer's bloom filter hit already. +func (dl *diffLayer) accountRLP(hash common.Hash, depth int) ([]byte, error) { + dl.lock.RLock() + defer dl.lock.RUnlock() + + // If the layer was flattened into, consider it invalid (any live reference to + // the original should be marked as unusable). + if dl.Stale() { + return nil, ErrSnapshotStale + } + // If the account is known locally, return it + if data, ok := dl.accountData[hash]; ok { + snapshotDirtyAccountHitMeter.Mark(1) + snapshotDirtyAccountHitDepthHist.Update(int64(depth)) + snapshotDirtyAccountReadMeter.Mark(int64(len(data))) + snapshotBloomAccountTrueHitMeter.Mark(1) + return data, nil + } + // If the account is known locally, but deleted, return it + if _, ok := dl.destructSet[hash]; ok { + snapshotDirtyAccountHitMeter.Mark(1) + snapshotDirtyAccountHitDepthHist.Update(int64(depth)) + snapshotDirtyAccountInexMeter.Mark(1) + snapshotBloomAccountTrueHitMeter.Mark(1) + return nil, nil + } + // Account unknown to this diff, resolve from parent + if diff, ok := dl.parent.(*diffLayer); ok { + return diff.accountRLP(hash, depth+1) + } + // Failed to resolve through diff layers, mark a bloom error and use the disk + snapshotBloomAccountFalseHitMeter.Mark(1) + return dl.parent.AccountRLP(hash) +} + +// Storage directly retrieves the storage data associated with a particular hash, +// within a particular account. If the slot is unknown to this diff, it's parent +// is consulted. +// +// Note the returned slot is not a copy, please don't modify it. +func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, error) { + // Check the bloom filter first whether there's even a point in reaching into + // all the maps in all the layers below + dl.lock.RLock() + hit := dl.diffed.Contains(storageBloomHasher{accountHash, storageHash}) + if !hit { + hit = dl.diffed.Contains(destructBloomHasher(accountHash)) + } + var origin *diskLayer + if !hit { + origin = dl.origin // extract origin while holding the lock + } + dl.lock.RUnlock() + + // If the bloom filter misses, don't even bother with traversing the memory + // diff layers, reach straight into the bottom persistent disk layer + if origin != nil { + snapshotBloomStorageMissMeter.Mark(1) + return origin.Storage(accountHash, storageHash) + } + // The bloom filter hit, start poking in the internal maps + return dl.storage(accountHash, storageHash, 0) +} + +// storage is an internal version of Storage that skips the bloom filter checks +// and uses the internal maps to try and retrieve the data. It's meant to be +// used if a higher layer's bloom filter hit already. +func (dl *diffLayer) storage(accountHash, storageHash common.Hash, depth int) ([]byte, error) { + dl.lock.RLock() + defer dl.lock.RUnlock() + + // If the layer was flattened into, consider it invalid (any live reference to + // the original should be marked as unusable). + if dl.Stale() { + return nil, ErrSnapshotStale + } + // If the account is known locally, try to resolve the slot locally + if storage, ok := dl.storageData[accountHash]; ok { + if data, ok := storage[storageHash]; ok { + snapshotDirtyStorageHitMeter.Mark(1) + snapshotDirtyStorageHitDepthHist.Update(int64(depth)) + if n := len(data); n > 0 { + snapshotDirtyStorageReadMeter.Mark(int64(n)) + } else { + snapshotDirtyStorageInexMeter.Mark(1) + } + snapshotBloomStorageTrueHitMeter.Mark(1) + return data, nil + } + } + // If the account is known locally, but deleted, return an empty slot + if _, ok := dl.destructSet[accountHash]; ok { + snapshotDirtyStorageHitMeter.Mark(1) + snapshotDirtyStorageHitDepthHist.Update(int64(depth)) + snapshotDirtyStorageInexMeter.Mark(1) + snapshotBloomStorageTrueHitMeter.Mark(1) + return nil, nil + } + // Storage slot unknown to this diff, resolve from parent + if diff, ok := dl.parent.(*diffLayer); ok { + return diff.storage(accountHash, storageHash, depth+1) + } + // Failed to resolve through diff layers, mark a bloom error and use the disk + snapshotBloomStorageFalseHitMeter.Mark(1) + return dl.parent.Storage(accountHash, storageHash) +} + +// Update creates a new layer on top of the existing snapshot diff tree with +// the specified data items. +func (dl *diffLayer) Update(blockRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { + return newDiffLayer(dl, blockRoot, destructs, accounts, storage) +} + +// flatten pushes all data from this point downwards, flattening everything into +// a single diff at the bottom. Since usually the lowermost diff is the largest, +// the flattening builds up from there in reverse. +func (dl *diffLayer) flatten() snapshot { + // If the parent is not diff, we're the first in line, return unmodified + parent, ok := dl.parent.(*diffLayer) + if !ok { + return dl + } + // Parent is a diff, flatten it first (note, apart from weird corned cases, + // flatten will realistically only ever merge 1 layer, so there's no need to + // be smarter about grouping flattens together). + parent = parent.flatten().(*diffLayer) + + parent.lock.Lock() + defer parent.lock.Unlock() + + // Before actually writing all our data to the parent, first ensure that the + // parent hasn't been 'corrupted' by someone else already flattening into it + if atomic.SwapUint32(&parent.stale, 1) != 0 { + panic("parent diff layer is stale") // we've flattened into the same parent from two children, boo + } + // Overwrite all the updated accounts blindly, merge the sorted list + for hash := range dl.destructSet { + parent.destructSet[hash] = struct{}{} + delete(parent.accountData, hash) + delete(parent.storageData, hash) + } + for hash, data := range dl.accountData { + parent.accountData[hash] = data + } + // Overwrite all the updated storage slots (individually) + for accountHash, storage := range dl.storageData { + // If storage didn't exist (or was deleted) in the parent, overwrite blindly + if _, ok := parent.storageData[accountHash]; !ok { + parent.storageData[accountHash] = storage + continue + } + // Storage exists in both parent and child, merge the slots + comboData := parent.storageData[accountHash] + for storageHash, data := range storage { + comboData[storageHash] = data + } + } + // Return the combo parent + return &diffLayer{ + parent: parent.parent, + origin: parent.origin, + root: dl.root, + destructSet: parent.destructSet, + accountData: parent.accountData, + storageData: parent.storageData, + storageList: make(map[common.Hash][]common.Hash), + diffed: dl.diffed, + memory: parent.memory + dl.memory, + } +} + +// AccountList returns a sorted list of all accounts in this diffLayer, including +// the deleted ones. +// +// Note, the returned slice is not a copy, so do not modify it. +func (dl *diffLayer) AccountList() []common.Hash { + // If an old list already exists, return it + dl.lock.RLock() + list := dl.accountList + dl.lock.RUnlock() + + if list != nil { + return list + } + // No old sorted account list exists, generate a new one + dl.lock.Lock() + defer dl.lock.Unlock() + + dl.accountList = make([]common.Hash, 0, len(dl.destructSet)+len(dl.accountData)) + for hash := range dl.accountData { + dl.accountList = append(dl.accountList, hash) + } + for hash := range dl.destructSet { + if _, ok := dl.accountData[hash]; !ok { + dl.accountList = append(dl.accountList, hash) + } + } + sort.Sort(hashes(dl.accountList)) + dl.memory += uint64(len(dl.accountList) * common.HashLength) + return dl.accountList +} + +// StorageList returns a sorted list of all storage slot hashes in this diffLayer +// for the given account. If the whole storage is destructed in this layer, then +// an additional flag *destructed = true* will be returned, otherwise the flag is +// false. Besides, the returned list will include the hash of deleted storage slot. +// Note a special case is an account is deleted in a prior tx but is recreated in +// the following tx with some storage slots set. In this case the returned list is +// not empty but the flag is true. +// +// Note, the returned slice is not a copy, so do not modify it. +func (dl *diffLayer) StorageList(accountHash common.Hash) ([]common.Hash, bool) { + dl.lock.RLock() + _, destructed := dl.destructSet[accountHash] + if _, ok := dl.storageData[accountHash]; !ok { + // Account not tracked by this layer + dl.lock.RUnlock() + return nil, destructed + } + // If an old list already exists, return it + if list, exist := dl.storageList[accountHash]; exist { + dl.lock.RUnlock() + return list, destructed // the cached list can't be nil + } + dl.lock.RUnlock() + + // No old sorted account list exists, generate a new one + dl.lock.Lock() + defer dl.lock.Unlock() + + storageMap := dl.storageData[accountHash] + storageList := make([]common.Hash, 0, len(storageMap)) + for k := range storageMap { + storageList = append(storageList, k) + } + sort.Sort(hashes(storageList)) + dl.storageList[accountHash] = storageList + dl.memory += uint64(len(dl.storageList)*common.HashLength + common.HashLength) + return storageList, destructed +} diff --git a/core/state/snapshot/difflayer_test.go b/core/state/snapshot/difflayer_test.go new file mode 100644 index 0000000000..674a031b16 --- /dev/null +++ b/core/state/snapshot/difflayer_test.go @@ -0,0 +1,399 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "bytes" + crand "crypto/rand" + "math/rand" + "testing" + + "github.com/VictoriaMetrics/fastcache" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb/memorydb" +) + +func copyDestructs(destructs map[common.Hash]struct{}) map[common.Hash]struct{} { + copy := make(map[common.Hash]struct{}) + for hash := range destructs { + copy[hash] = struct{}{} + } + return copy +} + +func copyAccounts(accounts map[common.Hash][]byte) map[common.Hash][]byte { + copy := make(map[common.Hash][]byte) + for hash, blob := range accounts { + copy[hash] = blob + } + return copy +} + +func copyStorage(storage map[common.Hash]map[common.Hash][]byte) map[common.Hash]map[common.Hash][]byte { + copy := make(map[common.Hash]map[common.Hash][]byte) + for accHash, slots := range storage { + copy[accHash] = make(map[common.Hash][]byte) + for slotHash, blob := range slots { + copy[accHash][slotHash] = blob + } + } + return copy +} + +// TestMergeBasics tests some simple merges +func TestMergeBasics(t *testing.T) { + var ( + destructs = make(map[common.Hash]struct{}) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) + ) + // Fill up a parent + for i := 0; i < 100; i++ { + h := randomHash() + data := randomAccount() + + accounts[h] = data + if rand.Intn(4) == 0 { + destructs[h] = struct{}{} + } + if rand.Intn(2) == 0 { + accStorage := make(map[common.Hash][]byte) + value := make([]byte, 32) + crand.Read(value) + accStorage[randomHash()] = value + storage[h] = accStorage + } + } + // Add some (identical) layers on top + parent := newDiffLayer(emptyLayer(), common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) + child := newDiffLayer(parent, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) + child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) + child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) + child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) + // And flatten + merged := (child.flatten()).(*diffLayer) + + { // Check account lists + if have, want := len(merged.accountList), 0; have != want { + t.Errorf("accountList wrong: have %v, want %v", have, want) + } + if have, want := len(merged.AccountList()), len(accounts); have != want { + t.Errorf("AccountList() wrong: have %v, want %v", have, want) + } + if have, want := len(merged.accountList), len(accounts); have != want { + t.Errorf("accountList [2] wrong: have %v, want %v", have, want) + } + } + { // Check account drops + if have, want := len(merged.destructSet), len(destructs); have != want { + t.Errorf("accountDrop wrong: have %v, want %v", have, want) + } + } + { // Check storage lists + i := 0 + for aHash, sMap := range storage { + if have, want := len(merged.storageList), i; have != want { + t.Errorf("[1] storageList wrong: have %v, want %v", have, want) + } + list, _ := merged.StorageList(aHash) + if have, want := len(list), len(sMap); have != want { + t.Errorf("[2] StorageList() wrong: have %v, want %v", have, want) + } + if have, want := len(merged.storageList[aHash]), len(sMap); have != want { + t.Errorf("storageList wrong: have %v, want %v", have, want) + } + i++ + } + } +} + +// TestMergeDelete tests some deletion +func TestMergeDelete(t *testing.T) { + var ( + storage = make(map[common.Hash]map[common.Hash][]byte) + ) + // Fill up a parent + h1 := common.HexToHash("0x01") + h2 := common.HexToHash("0x02") + + flipDrops := func() map[common.Hash]struct{} { + return map[common.Hash]struct{}{ + h2: {}, + } + } + flipAccs := func() map[common.Hash][]byte { + return map[common.Hash][]byte{ + h1: randomAccount(), + } + } + flopDrops := func() map[common.Hash]struct{} { + return map[common.Hash]struct{}{ + h1: {}, + } + } + flopAccs := func() map[common.Hash][]byte { + return map[common.Hash][]byte{ + h2: randomAccount(), + } + } + // Add some flipAccs-flopping layers on top + parent := newDiffLayer(emptyLayer(), common.Hash{}, flipDrops(), flipAccs(), storage) + child := parent.Update(common.Hash{}, flopDrops(), flopAccs(), storage) + child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage) + child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage) + child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage) + child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage) + child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage) + + if data, _ := child.Account(h1); data == nil { + t.Errorf("last diff layer: expected %x account to be non-nil", h1) + } + if data, _ := child.Account(h2); data != nil { + t.Errorf("last diff layer: expected %x account to be nil", h2) + } + if _, ok := child.destructSet[h1]; ok { + t.Errorf("last diff layer: expected %x drop to be missing", h1) + } + if _, ok := child.destructSet[h2]; !ok { + t.Errorf("last diff layer: expected %x drop to be present", h1) + } + // And flatten + merged := (child.flatten()).(*diffLayer) + + if data, _ := merged.Account(h1); data == nil { + t.Errorf("merged layer: expected %x account to be non-nil", h1) + } + if data, _ := merged.Account(h2); data != nil { + t.Errorf("merged layer: expected %x account to be nil", h2) + } + if _, ok := merged.destructSet[h1]; !ok { // Note, drops stay alive until persisted to disk! + t.Errorf("merged diff layer: expected %x drop to be present", h1) + } + if _, ok := merged.destructSet[h2]; !ok { // Note, drops stay alive until persisted to disk! + t.Errorf("merged diff layer: expected %x drop to be present", h1) + } + // If we add more granular metering of memory, we can enable this again, + // but it's not implemented for now + //if have, want := merged.memory, child.memory; have != want { + // t.Errorf("mem wrong: have %d, want %d", have, want) + //} +} + +// This tests that if we create a new account, and set a slot, and then merge +// it, the lists will be correct. +func TestInsertAndMerge(t *testing.T) { + // Fill up a parent + var ( + acc = common.HexToHash("0x01") + slot = common.HexToHash("0x02") + parent *diffLayer + child *diffLayer + ) + { + var ( + destructs = make(map[common.Hash]struct{}) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) + ) + parent = newDiffLayer(emptyLayer(), common.Hash{}, destructs, accounts, storage) + } + { + var ( + destructs = make(map[common.Hash]struct{}) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) + ) + accounts[acc] = randomAccount() + storage[acc] = make(map[common.Hash][]byte) + storage[acc][slot] = []byte{0x01} + child = newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + } + // And flatten + merged := (child.flatten()).(*diffLayer) + { // Check that slot value is present + have, _ := merged.Storage(acc, slot) + if want := []byte{0x01}; !bytes.Equal(have, want) { + t.Errorf("merged slot value wrong: have %x, want %x", have, want) + } + } +} + +func emptyLayer() *diskLayer { + return &diskLayer{ + diskdb: memorydb.New(), + cache: fastcache.New(500 * 1024), + } +} + +// BenchmarkSearch checks how long it takes to find a non-existing key +// BenchmarkSearch-6 200000 10481 ns/op (1K per layer) +// BenchmarkSearch-6 200000 10760 ns/op (10K per layer) +// BenchmarkSearch-6 100000 17866 ns/op +// +// BenchmarkSearch-6 500000 3723 ns/op (10k per layer, only top-level RLock() +func BenchmarkSearch(b *testing.B) { + // First, we set up 128 diff layers, with 1K items each + fill := func(parent snapshot) *diffLayer { + var ( + destructs = make(map[common.Hash]struct{}) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) + ) + for i := 0; i < 10000; i++ { + accounts[randomHash()] = randomAccount() + } + return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + } + var layer snapshot + layer = emptyLayer() + for i := 0; i < 128; i++ { + layer = fill(layer) + } + key := crypto.Keccak256Hash([]byte{0x13, 0x38}) + b.ResetTimer() + for i := 0; i < b.N; i++ { + layer.AccountRLP(key) + } +} + +// BenchmarkSearchSlot checks how long it takes to find a non-existing key +// - Number of layers: 128 +// - Each layers contains the account, with a couple of storage slots +// BenchmarkSearchSlot-6 100000 14554 ns/op +// BenchmarkSearchSlot-6 100000 22254 ns/op (when checking parent root using mutex) +// BenchmarkSearchSlot-6 100000 14551 ns/op (when checking parent number using atomic) +// With bloom filter: +// BenchmarkSearchSlot-6 3467835 351 ns/op +func BenchmarkSearchSlot(b *testing.B) { + // First, we set up 128 diff layers, with 1K items each + accountKey := crypto.Keccak256Hash([]byte{0x13, 0x37}) + storageKey := crypto.Keccak256Hash([]byte{0x13, 0x37}) + accountRLP := randomAccount() + fill := func(parent snapshot) *diffLayer { + var ( + destructs = make(map[common.Hash]struct{}) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) + ) + accounts[accountKey] = accountRLP + + accStorage := make(map[common.Hash][]byte) + for i := 0; i < 5; i++ { + value := make([]byte, 32) + crand.Read(value) + accStorage[randomHash()] = value + storage[accountKey] = accStorage + } + return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + } + var layer snapshot + layer = emptyLayer() + for i := 0; i < 128; i++ { + layer = fill(layer) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + layer.Storage(accountKey, storageKey) + } +} + +// With accountList and sorting +// BenchmarkFlatten-6 50 29890856 ns/op +// +// Without sorting and tracking accountList +// BenchmarkFlatten-6 300 5511511 ns/op +func BenchmarkFlatten(b *testing.B) { + fill := func(parent snapshot) *diffLayer { + var ( + destructs = make(map[common.Hash]struct{}) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) + ) + for i := 0; i < 100; i++ { + accountKey := randomHash() + accounts[accountKey] = randomAccount() + + accStorage := make(map[common.Hash][]byte) + for i := 0; i < 20; i++ { + value := make([]byte, 32) + crand.Read(value) + accStorage[randomHash()] = value + } + storage[accountKey] = accStorage + } + return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + var layer snapshot + layer = emptyLayer() + for i := 1; i < 128; i++ { + layer = fill(layer) + } + b.StartTimer() + + for i := 1; i < 128; i++ { + dl, ok := layer.(*diffLayer) + if !ok { + break + } + layer = dl.flatten() + } + b.StopTimer() + } +} + +// This test writes ~324M of diff layers to disk, spread over +// - 128 individual layers, +// - each with 200 accounts +// - containing 200 slots +// +// BenchmarkJournal-6 1 1471373923 ns/ops +// BenchmarkJournal-6 1 1208083335 ns/op // bufio writer +func BenchmarkJournal(b *testing.B) { + fill := func(parent snapshot) *diffLayer { + var ( + destructs = make(map[common.Hash]struct{}) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) + ) + for i := 0; i < 200; i++ { + accountKey := randomHash() + accounts[accountKey] = randomAccount() + + accStorage := make(map[common.Hash][]byte) + for i := 0; i < 200; i++ { + value := make([]byte, 32) + crand.Read(value) + accStorage[randomHash()] = value + } + storage[accountKey] = accStorage + } + return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + } + layer := snapshot(emptyLayer()) + for i := 1; i < 128; i++ { + layer = fill(layer) + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + layer.Journal(new(bytes.Buffer)) + } +} diff --git a/core/state/snapshot/disklayer.go b/core/state/snapshot/disklayer.go new file mode 100644 index 0000000000..39879e1433 --- /dev/null +++ b/core/state/snapshot/disklayer.go @@ -0,0 +1,166 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "bytes" + "sync" + + "github.com/VictoriaMetrics/fastcache" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/harmony-one/harmony/core/rawdb" +) + +// diskLayer is a low level persistent snapshot built on top of a key-value store. +type diskLayer struct { + diskdb ethdb.KeyValueStore // Key-value store containing the base snapshot + triedb *trie.Database // Trie node cache for reconstruction purposes + cache *fastcache.Cache // Cache to avoid hitting the disk for direct access + + root common.Hash // Root hash of the base snapshot + stale bool // Signals that the layer became stale (state progressed) + + genMarker []byte // Marker for the state that's indexed during initial layer generation + genPending chan struct{} // Notification channel when generation is done (test synchronicity) + genAbort chan chan *generatorStats // Notification channel to abort generating the snapshot in this layer + + lock sync.RWMutex +} + +// Root returns root hash for which this snapshot was made. +func (dl *diskLayer) Root() common.Hash { + return dl.root +} + +// Parent always returns nil as there's no layer below the disk. +func (dl *diskLayer) Parent() snapshot { + return nil +} + +// Stale return whether this layer has become stale (was flattened across) or if +// it's still live. +func (dl *diskLayer) Stale() bool { + dl.lock.RLock() + defer dl.lock.RUnlock() + + return dl.stale +} + +// Account directly retrieves the account associated with a particular hash in +// the snapshot slim data format. +func (dl *diskLayer) Account(hash common.Hash) (*Account, error) { + data, err := dl.AccountRLP(hash) + if err != nil { + return nil, err + } + if len(data) == 0 { // can be both nil and []byte{} + return nil, nil + } + account := new(Account) + if err := rlp.DecodeBytes(data, account); err != nil { + panic(err) + } + return account, nil +} + +// AccountRLP directly retrieves the account RLP associated with a particular +// hash in the snapshot slim data format. +func (dl *diskLayer) AccountRLP(hash common.Hash) ([]byte, error) { + dl.lock.RLock() + defer dl.lock.RUnlock() + + // If the layer was flattened into, consider it invalid (any live reference to + // the original should be marked as unusable). + if dl.stale { + return nil, ErrSnapshotStale + } + // If the layer is being generated, ensure the requested hash has already been + // covered by the generator. + if dl.genMarker != nil && bytes.Compare(hash[:], dl.genMarker) > 0 { + return nil, ErrNotCoveredYet + } + // If we're in the disk layer, all diff layers missed + snapshotDirtyAccountMissMeter.Mark(1) + + // Try to retrieve the account from the memory cache + if blob, found := dl.cache.HasGet(nil, hash[:]); found { + snapshotCleanAccountHitMeter.Mark(1) + snapshotCleanAccountReadMeter.Mark(int64(len(blob))) + return blob, nil + } + // Cache doesn't contain account, pull from disk and cache for later + blob := rawdb.ReadAccountSnapshot(dl.diskdb, hash) + dl.cache.Set(hash[:], blob) + + snapshotCleanAccountMissMeter.Mark(1) + if n := len(blob); n > 0 { + snapshotCleanAccountWriteMeter.Mark(int64(n)) + } else { + snapshotCleanAccountInexMeter.Mark(1) + } + return blob, nil +} + +// Storage directly retrieves the storage data associated with a particular hash, +// within a particular account. +func (dl *diskLayer) Storage(accountHash, storageHash common.Hash) ([]byte, error) { + dl.lock.RLock() + defer dl.lock.RUnlock() + + // If the layer was flattened into, consider it invalid (any live reference to + // the original should be marked as unusable). + if dl.stale { + return nil, ErrSnapshotStale + } + key := append(accountHash[:], storageHash[:]...) + + // If the layer is being generated, ensure the requested hash has already been + // covered by the generator. + if dl.genMarker != nil && bytes.Compare(key, dl.genMarker) > 0 { + return nil, ErrNotCoveredYet + } + // If we're in the disk layer, all diff layers missed + snapshotDirtyStorageMissMeter.Mark(1) + + // Try to retrieve the storage slot from the memory cache + if blob, found := dl.cache.HasGet(nil, key); found { + snapshotCleanStorageHitMeter.Mark(1) + snapshotCleanStorageReadMeter.Mark(int64(len(blob))) + return blob, nil + } + // Cache doesn't contain storage slot, pull from disk and cache for later + blob := rawdb.ReadStorageSnapshot(dl.diskdb, accountHash, storageHash) + dl.cache.Set(key, blob) + + snapshotCleanStorageMissMeter.Mark(1) + if n := len(blob); n > 0 { + snapshotCleanStorageWriteMeter.Mark(int64(n)) + } else { + snapshotCleanStorageInexMeter.Mark(1) + } + return blob, nil +} + +// Update creates a new layer on top of the existing snapshot diff tree with +// the specified data items. Note, the maps are retained by the method to avoid +// copying everything. +func (dl *diskLayer) Update(blockHash common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { + return newDiffLayer(dl, blockHash, destructs, accounts, storage) +} diff --git a/core/state/snapshot/disklayer_test.go b/core/state/snapshot/disklayer_test.go new file mode 100644 index 0000000000..80070d619d --- /dev/null +++ b/core/state/snapshot/disklayer_test.go @@ -0,0 +1,574 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "bytes" + "testing" + + "github.com/VictoriaMetrics/fastcache" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/rlp" + "github.com/harmony-one/harmony/core/rawdb" +) + +// reverse reverses the contents of a byte slice. It's used to update random accs +// with deterministic changes. +func reverse(blob []byte) []byte { + res := make([]byte, len(blob)) + for i, b := range blob { + res[len(blob)-1-i] = b + } + return res +} + +// Tests that merging something into a disk layer persists it into the database +// and invalidates any previously written and cached values. +func TestDiskMerge(t *testing.T) { + // Create some accounts in the disk layer + db := memorydb.New() + + var ( + accNoModNoCache = common.Hash{0x1} + accNoModCache = common.Hash{0x2} + accModNoCache = common.Hash{0x3} + accModCache = common.Hash{0x4} + accDelNoCache = common.Hash{0x5} + accDelCache = common.Hash{0x6} + conNoModNoCache = common.Hash{0x7} + conNoModNoCacheSlot = common.Hash{0x70} + conNoModCache = common.Hash{0x8} + conNoModCacheSlot = common.Hash{0x80} + conModNoCache = common.Hash{0x9} + conModNoCacheSlot = common.Hash{0x90} + conModCache = common.Hash{0xa} + conModCacheSlot = common.Hash{0xa0} + conDelNoCache = common.Hash{0xb} + conDelNoCacheSlot = common.Hash{0xb0} + conDelCache = common.Hash{0xc} + conDelCacheSlot = common.Hash{0xc0} + conNukeNoCache = common.Hash{0xd} + conNukeNoCacheSlot = common.Hash{0xd0} + conNukeCache = common.Hash{0xe} + conNukeCacheSlot = common.Hash{0xe0} + baseRoot = randomHash() + diffRoot = randomHash() + ) + + rawdb.WriteAccountSnapshot(db, accNoModNoCache, accNoModNoCache[:]) + rawdb.WriteAccountSnapshot(db, accNoModCache, accNoModCache[:]) + rawdb.WriteAccountSnapshot(db, accModNoCache, accModNoCache[:]) + rawdb.WriteAccountSnapshot(db, accModCache, accModCache[:]) + rawdb.WriteAccountSnapshot(db, accDelNoCache, accDelNoCache[:]) + rawdb.WriteAccountSnapshot(db, accDelCache, accDelCache[:]) + + rawdb.WriteAccountSnapshot(db, conNoModNoCache, conNoModNoCache[:]) + rawdb.WriteStorageSnapshot(db, conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:]) + rawdb.WriteAccountSnapshot(db, conNoModCache, conNoModCache[:]) + rawdb.WriteStorageSnapshot(db, conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:]) + rawdb.WriteAccountSnapshot(db, conModNoCache, conModNoCache[:]) + rawdb.WriteStorageSnapshot(db, conModNoCache, conModNoCacheSlot, conModNoCacheSlot[:]) + rawdb.WriteAccountSnapshot(db, conModCache, conModCache[:]) + rawdb.WriteStorageSnapshot(db, conModCache, conModCacheSlot, conModCacheSlot[:]) + rawdb.WriteAccountSnapshot(db, conDelNoCache, conDelNoCache[:]) + rawdb.WriteStorageSnapshot(db, conDelNoCache, conDelNoCacheSlot, conDelNoCacheSlot[:]) + rawdb.WriteAccountSnapshot(db, conDelCache, conDelCache[:]) + rawdb.WriteStorageSnapshot(db, conDelCache, conDelCacheSlot, conDelCacheSlot[:]) + + rawdb.WriteAccountSnapshot(db, conNukeNoCache, conNukeNoCache[:]) + rawdb.WriteStorageSnapshot(db, conNukeNoCache, conNukeNoCacheSlot, conNukeNoCacheSlot[:]) + rawdb.WriteAccountSnapshot(db, conNukeCache, conNukeCache[:]) + rawdb.WriteStorageSnapshot(db, conNukeCache, conNukeCacheSlot, conNukeCacheSlot[:]) + + rawdb.WriteSnapshotRoot(db, baseRoot) + + // Create a disk layer based on the above and cache in some data + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + baseRoot: &diskLayer{ + diskdb: db, + cache: fastcache.New(500 * 1024), + root: baseRoot, + }, + }, + } + base := snaps.Snapshot(baseRoot) + base.AccountRLP(accNoModCache) + base.AccountRLP(accModCache) + base.AccountRLP(accDelCache) + base.Storage(conNoModCache, conNoModCacheSlot) + base.Storage(conModCache, conModCacheSlot) + base.Storage(conDelCache, conDelCacheSlot) + base.Storage(conNukeCache, conNukeCacheSlot) + + // Modify or delete some accounts, flatten everything onto disk + if err := snaps.Update(diffRoot, baseRoot, map[common.Hash]struct{}{ + accDelNoCache: {}, + accDelCache: {}, + conNukeNoCache: {}, + conNukeCache: {}, + }, map[common.Hash][]byte{ + accModNoCache: reverse(accModNoCache[:]), + accModCache: reverse(accModCache[:]), + }, map[common.Hash]map[common.Hash][]byte{ + conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])}, + conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])}, + conDelNoCache: {conDelNoCacheSlot: nil}, + conDelCache: {conDelCacheSlot: nil}, + }); err != nil { + t.Fatalf("failed to update snapshot tree: %v", err) + } + if err := snaps.Cap(diffRoot, 0); err != nil { + t.Fatalf("failed to flatten snapshot tree: %v", err) + } + // Retrieve all the data through the disk layer and validate it + base = snaps.Snapshot(diffRoot) + if _, ok := base.(*diskLayer); !ok { + t.Fatalf("update not flattend into the disk layer") + } + + // assertAccount ensures that an account matches the given blob. + assertAccount := func(account common.Hash, data []byte) { + t.Helper() + blob, err := base.AccountRLP(account) + if err != nil { + t.Errorf("account access (%x) failed: %v", account, err) + } else if !bytes.Equal(blob, data) { + t.Errorf("account access (%x) mismatch: have %x, want %x", account, blob, data) + } + } + assertAccount(accNoModNoCache, accNoModNoCache[:]) + assertAccount(accNoModCache, accNoModCache[:]) + assertAccount(accModNoCache, reverse(accModNoCache[:])) + assertAccount(accModCache, reverse(accModCache[:])) + assertAccount(accDelNoCache, nil) + assertAccount(accDelCache, nil) + + // assertStorage ensures that a storage slot matches the given blob. + assertStorage := func(account common.Hash, slot common.Hash, data []byte) { + t.Helper() + blob, err := base.Storage(account, slot) + if err != nil { + t.Errorf("storage access (%x:%x) failed: %v", account, slot, err) + } else if !bytes.Equal(blob, data) { + t.Errorf("storage access (%x:%x) mismatch: have %x, want %x", account, slot, blob, data) + } + } + assertStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:]) + assertStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:]) + assertStorage(conModNoCache, conModNoCacheSlot, reverse(conModNoCacheSlot[:])) + assertStorage(conModCache, conModCacheSlot, reverse(conModCacheSlot[:])) + assertStorage(conDelNoCache, conDelNoCacheSlot, nil) + assertStorage(conDelCache, conDelCacheSlot, nil) + assertStorage(conNukeNoCache, conNukeNoCacheSlot, nil) + assertStorage(conNukeCache, conNukeCacheSlot, nil) + + // Retrieve all the data directly from the database and validate it + + // assertDatabaseAccount ensures that an account from the database matches the given blob. + assertDatabaseAccount := func(account common.Hash, data []byte) { + t.Helper() + if blob := rawdb.ReadAccountSnapshot(db, account); !bytes.Equal(blob, data) { + t.Errorf("account database access (%x) mismatch: have %x, want %x", account, blob, data) + } + } + assertDatabaseAccount(accNoModNoCache, accNoModNoCache[:]) + assertDatabaseAccount(accNoModCache, accNoModCache[:]) + assertDatabaseAccount(accModNoCache, reverse(accModNoCache[:])) + assertDatabaseAccount(accModCache, reverse(accModCache[:])) + assertDatabaseAccount(accDelNoCache, nil) + assertDatabaseAccount(accDelCache, nil) + + // assertDatabaseStorage ensures that a storage slot from the database matches the given blob. + assertDatabaseStorage := func(account common.Hash, slot common.Hash, data []byte) { + t.Helper() + if blob := rawdb.ReadStorageSnapshot(db, account, slot); !bytes.Equal(blob, data) { + t.Errorf("storage database access (%x:%x) mismatch: have %x, want %x", account, slot, blob, data) + } + } + assertDatabaseStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:]) + assertDatabaseStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:]) + assertDatabaseStorage(conModNoCache, conModNoCacheSlot, reverse(conModNoCacheSlot[:])) + assertDatabaseStorage(conModCache, conModCacheSlot, reverse(conModCacheSlot[:])) + assertDatabaseStorage(conDelNoCache, conDelNoCacheSlot, nil) + assertDatabaseStorage(conDelCache, conDelCacheSlot, nil) + assertDatabaseStorage(conNukeNoCache, conNukeNoCacheSlot, nil) + assertDatabaseStorage(conNukeCache, conNukeCacheSlot, nil) +} + +// Tests that merging something into a disk layer persists it into the database +// and invalidates any previously written and cached values, discarding anything +// after the in-progress generation marker. +func TestDiskPartialMerge(t *testing.T) { + // Iterate the test a few times to ensure we pick various internal orderings + // for the data slots as well as the progress marker. + for i := 0; i < 1024; i++ { + // Create some accounts in the disk layer + db := memorydb.New() + + var ( + accNoModNoCache = randomHash() + accNoModCache = randomHash() + accModNoCache = randomHash() + accModCache = randomHash() + accDelNoCache = randomHash() + accDelCache = randomHash() + conNoModNoCache = randomHash() + conNoModNoCacheSlot = randomHash() + conNoModCache = randomHash() + conNoModCacheSlot = randomHash() + conModNoCache = randomHash() + conModNoCacheSlot = randomHash() + conModCache = randomHash() + conModCacheSlot = randomHash() + conDelNoCache = randomHash() + conDelNoCacheSlot = randomHash() + conDelCache = randomHash() + conDelCacheSlot = randomHash() + conNukeNoCache = randomHash() + conNukeNoCacheSlot = randomHash() + conNukeCache = randomHash() + conNukeCacheSlot = randomHash() + baseRoot = randomHash() + diffRoot = randomHash() + genMarker = append(randomHash().Bytes(), randomHash().Bytes()...) + ) + + // insertAccount injects an account into the database if it's after the + // generator marker, drops the op otherwise. This is needed to seed the + // database with a valid starting snapshot. + insertAccount := func(account common.Hash, data []byte) { + if bytes.Compare(account[:], genMarker) <= 0 { + rawdb.WriteAccountSnapshot(db, account, data[:]) + } + } + insertAccount(accNoModNoCache, accNoModNoCache[:]) + insertAccount(accNoModCache, accNoModCache[:]) + insertAccount(accModNoCache, accModNoCache[:]) + insertAccount(accModCache, accModCache[:]) + insertAccount(accDelNoCache, accDelNoCache[:]) + insertAccount(accDelCache, accDelCache[:]) + + // insertStorage injects a storage slot into the database if it's after + // the generator marker, drops the op otherwise. This is needed to seed + // the database with a valid starting snapshot. + insertStorage := func(account common.Hash, slot common.Hash, data []byte) { + if bytes.Compare(append(account[:], slot[:]...), genMarker) <= 0 { + rawdb.WriteStorageSnapshot(db, account, slot, data[:]) + } + } + insertAccount(conNoModNoCache, conNoModNoCache[:]) + insertStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:]) + insertAccount(conNoModCache, conNoModCache[:]) + insertStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:]) + insertAccount(conModNoCache, conModNoCache[:]) + insertStorage(conModNoCache, conModNoCacheSlot, conModNoCacheSlot[:]) + insertAccount(conModCache, conModCache[:]) + insertStorage(conModCache, conModCacheSlot, conModCacheSlot[:]) + insertAccount(conDelNoCache, conDelNoCache[:]) + insertStorage(conDelNoCache, conDelNoCacheSlot, conDelNoCacheSlot[:]) + insertAccount(conDelCache, conDelCache[:]) + insertStorage(conDelCache, conDelCacheSlot, conDelCacheSlot[:]) + + insertAccount(conNukeNoCache, conNukeNoCache[:]) + insertStorage(conNukeNoCache, conNukeNoCacheSlot, conNukeNoCacheSlot[:]) + insertAccount(conNukeCache, conNukeCache[:]) + insertStorage(conNukeCache, conNukeCacheSlot, conNukeCacheSlot[:]) + + rawdb.WriteSnapshotRoot(db, baseRoot) + + // Create a disk layer based on the above using a random progress marker + // and cache in some data. + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + baseRoot: &diskLayer{ + diskdb: db, + cache: fastcache.New(500 * 1024), + root: baseRoot, + }, + }, + } + snaps.layers[baseRoot].(*diskLayer).genMarker = genMarker + base := snaps.Snapshot(baseRoot) + + // assertAccount ensures that an account matches the given blob if it's + // already covered by the disk snapshot, and errors out otherwise. + assertAccount := func(account common.Hash, data []byte) { + t.Helper() + blob, err := base.AccountRLP(account) + if bytes.Compare(account[:], genMarker) > 0 && err != ErrNotCoveredYet { + t.Fatalf("test %d: post-marker (%x) account access (%x) succeeded: %x", i, genMarker, account, blob) + } + if bytes.Compare(account[:], genMarker) <= 0 && !bytes.Equal(blob, data) { + t.Fatalf("test %d: pre-marker (%x) account access (%x) mismatch: have %x, want %x", i, genMarker, account, blob, data) + } + } + assertAccount(accNoModCache, accNoModCache[:]) + assertAccount(accModCache, accModCache[:]) + assertAccount(accDelCache, accDelCache[:]) + + // assertStorage ensures that a storage slot matches the given blob if + // it's already covered by the disk snapshot, and errors out otherwise. + assertStorage := func(account common.Hash, slot common.Hash, data []byte) { + t.Helper() + blob, err := base.Storage(account, slot) + if bytes.Compare(append(account[:], slot[:]...), genMarker) > 0 && err != ErrNotCoveredYet { + t.Fatalf("test %d: post-marker (%x) storage access (%x:%x) succeeded: %x", i, genMarker, account, slot, blob) + } + if bytes.Compare(append(account[:], slot[:]...), genMarker) <= 0 && !bytes.Equal(blob, data) { + t.Fatalf("test %d: pre-marker (%x) storage access (%x:%x) mismatch: have %x, want %x", i, genMarker, account, slot, blob, data) + } + } + assertStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:]) + assertStorage(conModCache, conModCacheSlot, conModCacheSlot[:]) + assertStorage(conDelCache, conDelCacheSlot, conDelCacheSlot[:]) + assertStorage(conNukeCache, conNukeCacheSlot, conNukeCacheSlot[:]) + + // Modify or delete some accounts, flatten everything onto disk + if err := snaps.Update(diffRoot, baseRoot, map[common.Hash]struct{}{ + accDelNoCache: {}, + accDelCache: {}, + conNukeNoCache: {}, + conNukeCache: {}, + }, map[common.Hash][]byte{ + accModNoCache: reverse(accModNoCache[:]), + accModCache: reverse(accModCache[:]), + }, map[common.Hash]map[common.Hash][]byte{ + conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])}, + conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])}, + conDelNoCache: {conDelNoCacheSlot: nil}, + conDelCache: {conDelCacheSlot: nil}, + }); err != nil { + t.Fatalf("test %d: failed to update snapshot tree: %v", i, err) + } + if err := snaps.Cap(diffRoot, 0); err != nil { + t.Fatalf("test %d: failed to flatten snapshot tree: %v", i, err) + } + // Retrieve all the data through the disk layer and validate it + base = snaps.Snapshot(diffRoot) + if _, ok := base.(*diskLayer); !ok { + t.Fatalf("test %d: update not flattend into the disk layer", i) + } + assertAccount(accNoModNoCache, accNoModNoCache[:]) + assertAccount(accNoModCache, accNoModCache[:]) + assertAccount(accModNoCache, reverse(accModNoCache[:])) + assertAccount(accModCache, reverse(accModCache[:])) + assertAccount(accDelNoCache, nil) + assertAccount(accDelCache, nil) + + assertStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:]) + assertStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:]) + assertStorage(conModNoCache, conModNoCacheSlot, reverse(conModNoCacheSlot[:])) + assertStorage(conModCache, conModCacheSlot, reverse(conModCacheSlot[:])) + assertStorage(conDelNoCache, conDelNoCacheSlot, nil) + assertStorage(conDelCache, conDelCacheSlot, nil) + assertStorage(conNukeNoCache, conNukeNoCacheSlot, nil) + assertStorage(conNukeCache, conNukeCacheSlot, nil) + + // Retrieve all the data directly from the database and validate it + + // assertDatabaseAccount ensures that an account inside the database matches + // the given blob if it's already covered by the disk snapshot, and does not + // exist otherwise. + assertDatabaseAccount := func(account common.Hash, data []byte) { + t.Helper() + blob := rawdb.ReadAccountSnapshot(db, account) + if bytes.Compare(account[:], genMarker) > 0 && blob != nil { + t.Fatalf("test %d: post-marker (%x) account database access (%x) succeeded: %x", i, genMarker, account, blob) + } + if bytes.Compare(account[:], genMarker) <= 0 && !bytes.Equal(blob, data) { + t.Fatalf("test %d: pre-marker (%x) account database access (%x) mismatch: have %x, want %x", i, genMarker, account, blob, data) + } + } + assertDatabaseAccount(accNoModNoCache, accNoModNoCache[:]) + assertDatabaseAccount(accNoModCache, accNoModCache[:]) + assertDatabaseAccount(accModNoCache, reverse(accModNoCache[:])) + assertDatabaseAccount(accModCache, reverse(accModCache[:])) + assertDatabaseAccount(accDelNoCache, nil) + assertDatabaseAccount(accDelCache, nil) + + // assertDatabaseStorage ensures that a storage slot inside the database + // matches the given blob if it's already covered by the disk snapshot, + // and does not exist otherwise. + assertDatabaseStorage := func(account common.Hash, slot common.Hash, data []byte) { + t.Helper() + blob := rawdb.ReadStorageSnapshot(db, account, slot) + if bytes.Compare(append(account[:], slot[:]...), genMarker) > 0 && blob != nil { + t.Fatalf("test %d: post-marker (%x) storage database access (%x:%x) succeeded: %x", i, genMarker, account, slot, blob) + } + if bytes.Compare(append(account[:], slot[:]...), genMarker) <= 0 && !bytes.Equal(blob, data) { + t.Fatalf("test %d: pre-marker (%x) storage database access (%x:%x) mismatch: have %x, want %x", i, genMarker, account, slot, blob, data) + } + } + assertDatabaseStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:]) + assertDatabaseStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:]) + assertDatabaseStorage(conModNoCache, conModNoCacheSlot, reverse(conModNoCacheSlot[:])) + assertDatabaseStorage(conModCache, conModCacheSlot, reverse(conModCacheSlot[:])) + assertDatabaseStorage(conDelNoCache, conDelNoCacheSlot, nil) + assertDatabaseStorage(conDelCache, conDelCacheSlot, nil) + assertDatabaseStorage(conNukeNoCache, conNukeNoCacheSlot, nil) + assertDatabaseStorage(conNukeCache, conNukeCacheSlot, nil) + } +} + +// Tests that when the bottom-most diff layer is merged into the disk +// layer whether the corresponding generator is persisted correctly. +func TestDiskGeneratorPersistence(t *testing.T) { + var ( + accOne = randomHash() + accTwo = randomHash() + accOneSlotOne = randomHash() + accOneSlotTwo = randomHash() + + accThree = randomHash() + accThreeSlot = randomHash() + baseRoot = randomHash() + diffRoot = randomHash() + diffTwoRoot = randomHash() + genMarker = append(randomHash().Bytes(), randomHash().Bytes()...) + ) + // Testing scenario 1, the disk layer is still under the construction. + db := rawdb.NewMemoryDatabase() + + rawdb.WriteAccountSnapshot(db, accOne, accOne[:]) + rawdb.WriteStorageSnapshot(db, accOne, accOneSlotOne, accOneSlotOne[:]) + rawdb.WriteStorageSnapshot(db, accOne, accOneSlotTwo, accOneSlotTwo[:]) + rawdb.WriteSnapshotRoot(db, baseRoot) + + // Create a disk layer based on all above updates + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + baseRoot: &diskLayer{ + diskdb: db, + cache: fastcache.New(500 * 1024), + root: baseRoot, + genMarker: genMarker, + }, + }, + } + // Modify or delete some accounts, flatten everything onto disk + if err := snaps.Update(diffRoot, baseRoot, nil, map[common.Hash][]byte{ + accTwo: accTwo[:], + }, nil); err != nil { + t.Fatalf("failed to update snapshot tree: %v", err) + } + if err := snaps.Cap(diffRoot, 0); err != nil { + t.Fatalf("failed to flatten snapshot tree: %v", err) + } + blob := rawdb.ReadSnapshotGenerator(db) + var generator journalGenerator + if err := rlp.DecodeBytes(blob, &generator); err != nil { + t.Fatalf("Failed to decode snapshot generator %v", err) + } + if !bytes.Equal(generator.Marker, genMarker) { + t.Fatalf("Generator marker is not matched") + } + // Test scenario 2, the disk layer is fully generated + // Modify or delete some accounts, flatten everything onto disk + if err := snaps.Update(diffTwoRoot, diffRoot, nil, map[common.Hash][]byte{ + accThree: accThree.Bytes(), + }, map[common.Hash]map[common.Hash][]byte{ + accThree: {accThreeSlot: accThreeSlot.Bytes()}, + }); err != nil { + t.Fatalf("failed to update snapshot tree: %v", err) + } + diskLayer := snaps.layers[snaps.diskRoot()].(*diskLayer) + diskLayer.genMarker = nil // Construction finished + if err := snaps.Cap(diffTwoRoot, 0); err != nil { + t.Fatalf("failed to flatten snapshot tree: %v", err) + } + blob = rawdb.ReadSnapshotGenerator(db) + if err := rlp.DecodeBytes(blob, &generator); err != nil { + t.Fatalf("Failed to decode snapshot generator %v", err) + } + if len(generator.Marker) != 0 { + t.Fatalf("Failed to update snapshot generator") + } +} + +// Tests that merging something into a disk layer persists it into the database +// and invalidates any previously written and cached values, discarding anything +// after the in-progress generation marker. +// +// This test case is a tiny specialized case of TestDiskPartialMerge, which tests +// some very specific cornercases that random tests won't ever trigger. +func TestDiskMidAccountPartialMerge(t *testing.T) { + // TODO(@karalabe) ? +} + +// TestDiskSeek tests that seek-operations work on the disk layer +func TestDiskSeek(t *testing.T) { + // Create some accounts in the disk layer + db := rawdb.NewMemoryDatabase() + defer db.Close() + + // Fill even keys [0,2,4...] + for i := 0; i < 0xff; i += 2 { + acc := common.Hash{byte(i)} + rawdb.WriteAccountSnapshot(db, acc, acc[:]) + } + // Add an 'higher' key, with incorrect (higher) prefix + highKey := []byte{rawdb.SnapshotAccountPrefix[0] + 1} + db.Put(highKey, []byte{0xff, 0xff}) + + baseRoot := randomHash() + rawdb.WriteSnapshotRoot(db, baseRoot) + + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + baseRoot: &diskLayer{ + diskdb: db, + cache: fastcache.New(500 * 1024), + root: baseRoot, + }, + }, + } + // Test some different seek positions + type testcase struct { + pos byte + expkey byte + } + var cases = []testcase{ + {0xff, 0x55}, // this should exit immediately without checking key + {0x01, 0x02}, + {0xfe, 0xfe}, + {0xfd, 0xfe}, + {0x00, 0x00}, + } + for i, tc := range cases { + it, err := snaps.AccountIterator(baseRoot, common.Hash{tc.pos}) + if err != nil { + t.Fatalf("case %d, error: %v", i, err) + } + count := 0 + for it.Next() { + k, v, err := it.Hash()[0], it.Account()[0], it.Error() + if err != nil { + t.Fatalf("test %d, item %d, error: %v", i, count, err) + } + // First item in iterator should have the expected key + if count == 0 && k != tc.expkey { + t.Fatalf("test %d, item %d, got %v exp %v", i, count, k, tc.expkey) + } + count++ + if v != k { + t.Fatalf("test %d, item %d, value wrong, got %v exp %v", i, count, v, k) + } + } + } +} diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go new file mode 100644 index 0000000000..e5376429bc --- /dev/null +++ b/core/state/snapshot/generate.go @@ -0,0 +1,756 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "bytes" + "errors" + "fmt" + "math/big" + "time" + + "github.com/VictoriaMetrics/fastcache" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/harmony-one/harmony/core/rawdb" + "github.com/harmony-one/harmony/internal/utils" +) + +var ( + // accountCheckRange is the upper limit of the number of accounts involved in + // each range check. This is a value estimated based on experience. If this + // range is too large, the failure rate of range proof will increase. Otherwise, + // if the range is too small, the efficiency of the state recovery will decrease. + accountCheckRange = 128 + + // storageCheckRange is the upper limit of the number of storage slots involved + // in each range check. This is a value estimated based on experience. If this + // range is too large, the failure rate of range proof will increase. Otherwise, + // if the range is too small, the efficiency of the state recovery will decrease. + storageCheckRange = 1024 + + // errMissingTrie is returned if the target trie is missing while the generation + // is running. In this case the generation is aborted and wait the new signal. + errMissingTrie = errors.New("missing trie") +) + +// generateSnapshot regenerates a brand new snapshot based on an existing state +// database and head block asynchronously. The snapshot is returned immediately +// and generation is continued in the background until done. +func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash) *diskLayer { + // Create a new disk layer with an initialized state marker at zero + var ( + stats = &generatorStats{start: time.Now()} + batch = diskdb.NewBatch() + genMarker = []byte{} // Initialized but empty! + ) + rawdb.WriteSnapshotRoot(batch, root) + journalProgress(batch, genMarker, stats) + if err := batch.Write(); err != nil { + utils.Logger().Fatal().Err(err).Msg("Failed to write initialized state marker") + } + base := &diskLayer{ + diskdb: diskdb, + triedb: triedb, + root: root, + cache: fastcache.New(cache * 1024 * 1024), + genMarker: genMarker, + genPending: make(chan struct{}), + genAbort: make(chan chan *generatorStats), + } + go base.generate(stats) + utils.Logger().Debug().Interface("root", root).Msg("Start snapshot generation") + return base +} + +// journalProgress persists the generator stats into the database to resume later. +func journalProgress(db ethdb.KeyValueWriter, marker []byte, stats *generatorStats) { + // Write out the generator marker. Note it's a standalone disk layer generator + // which is not mixed with journal. It's ok if the generator is persisted while + // journal is not. + entry := journalGenerator{ + Done: marker == nil, + Marker: marker, + } + if stats != nil { + entry.Accounts = stats.accounts + entry.Slots = stats.slots + entry.Storage = uint64(stats.storage) + } + blob, err := rlp.EncodeToBytes(entry) + if err != nil { + panic(err) // Cannot happen, here to catch dev errors + } + var logstr string + switch { + case marker == nil: + logstr = "done" + case bytes.Equal(marker, []byte{}): + logstr = "empty" + case len(marker) == common.HashLength: + logstr = fmt.Sprintf("%#x", marker) + default: + logstr = fmt.Sprintf("%#x:%#x", marker[:common.HashLength], marker[common.HashLength:]) + } + utils.Logger().Debug().Err(err).Str("progress", logstr).Msg("Journalled generator progress") + rawdb.WriteSnapshotGenerator(db, blob) +} + +// proofResult contains the output of range proving which can be used +// for further processing regardless if it is successful or not. +type proofResult struct { + keys [][]byte // The key set of all elements being iterated, even proving is failed + vals [][]byte // The val set of all elements being iterated, even proving is failed + diskMore bool // Set when the database has extra snapshot states since last iteration + trieMore bool // Set when the trie has extra snapshot states(only meaningful for successful proving) + proofErr error // Indicator whether the given state range is valid or not + tr *trie.Trie // The trie, in case the trie was resolved by the prover (may be nil) +} + +// valid returns the indicator that range proof is successful or not. +func (result *proofResult) valid() bool { + return result.proofErr == nil +} + +// last returns the last verified element key regardless of whether the range proof is +// successful or not. Nil is returned if nothing involved in the proving. +func (result *proofResult) last() []byte { + var last []byte + if len(result.keys) > 0 { + last = result.keys[len(result.keys)-1] + } + return last +} + +// forEach iterates all the visited elements and applies the given callback on them. +// The iteration is aborted if the callback returns non-nil error. +func (result *proofResult) forEach(callback func(key []byte, val []byte) error) error { + for i := 0; i < len(result.keys); i++ { + key, val := result.keys[i], result.vals[i] + if err := callback(key, val); err != nil { + return err + } + } + return nil +} + +// proveRange proves the snapshot segment with particular prefix is "valid". +// The iteration start point will be assigned if the iterator is restored from +// the last interruption. Max will be assigned in order to limit the maximum +// amount of data involved in each iteration. +// +// The proof result will be returned if the range proving is finished, otherwise +// the error will be returned to abort the entire procedure. +func (dl *diskLayer) proveRange(ctx *generatorContext, trieId *trie.ID, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) { + var ( + keys [][]byte + vals [][]byte + proof = rawdb.NewMemoryDatabase() + diskMore = false + iter = ctx.iterator(kind) + start = time.Now() + min = append(prefix, origin...) + ) + for iter.Next() { + // Ensure the iterated item is always equal or larger than the given origin. + key := iter.Key() + if bytes.Compare(key, min) < 0 { + return nil, errors.New("invalid iteration position") + } + // Ensure the iterated item still fall in the specified prefix. If + // not which means the items in the specified area are all visited. + // Move the iterator a step back since we iterate one extra element + // out. + if !bytes.Equal(key[:len(prefix)], prefix) { + iter.Hold() + break + } + // Break if we've reached the max size, and signal that we're not + // done yet. Move the iterator a step back since we iterate one + // extra element out. + if len(keys) == max { + iter.Hold() + diskMore = true + break + } + keys = append(keys, common.CopyBytes(key[len(prefix):])) + + if valueConvertFn == nil { + vals = append(vals, common.CopyBytes(iter.Value())) + } else { + val, err := valueConvertFn(iter.Value()) + if err != nil { + // Special case, the state data is corrupted (invalid slim-format account), + // don't abort the entire procedure directly. Instead, let the fallback + // generation to heal the invalid data. + // + // Here append the original value to ensure that the number of key and + // value are aligned. + vals = append(vals, common.CopyBytes(iter.Value())) + utils.Logger().Error().Err(err).Msg("Failed to convert account state data") + } else { + vals = append(vals, val) + } + } + } + // Update metrics for database iteration and merkle proving + if kind == snapStorage { + snapStorageSnapReadCounter.Inc(time.Since(start).Nanoseconds()) + } else { + snapAccountSnapReadCounter.Inc(time.Since(start).Nanoseconds()) + } + defer func(start time.Time) { + if kind == snapStorage { + snapStorageProveCounter.Inc(time.Since(start).Nanoseconds()) + } else { + snapAccountProveCounter.Inc(time.Since(start).Nanoseconds()) + } + }(time.Now()) + + // The snap state is exhausted, pass the entire key/val set for verification + root := trieId.Root + if origin == nil && !diskMore { + stackTr := trie.NewStackTrie(nil) + for i, key := range keys { + stackTr.TryUpdate(key, vals[i]) + } + if gotRoot := stackTr.Hash(); gotRoot != root { + return &proofResult{ + keys: keys, + vals: vals, + proofErr: fmt.Errorf("wrong root: have %#x want %#x", gotRoot, root), + }, nil + } + return &proofResult{keys: keys, vals: vals}, nil + } + // Snap state is chunked, generate edge proofs for verification. + tr, err := trie.New(trieId, dl.triedb) + if err != nil { + ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker) + return nil, errMissingTrie + } + // Firstly find out the key of last iterated element. + var last []byte + if len(keys) > 0 { + last = keys[len(keys)-1] + } + // Generate the Merkle proofs for the first and last element + if origin == nil { + origin = common.Hash{}.Bytes() + } + if err := tr.Prove(origin, 0, proof); err != nil { + utils.Logger().Debug().Err(err). + Msg("Failed to prove range") + + return &proofResult{ + keys: keys, + vals: vals, + diskMore: diskMore, + proofErr: err, + tr: tr, + }, nil + } + if last != nil { + if err := tr.Prove(last, 0, proof); err != nil { + utils.Logger().Debug().Err(err).Str("kind", kind).Bytes("last", last).Msg("Failed to prove range") + return &proofResult{ + keys: keys, + vals: vals, + diskMore: diskMore, + proofErr: err, + tr: tr, + }, nil + } + } + // Verify the snapshot segment with range prover, ensure that all flat states + // in this range correspond to merkle trie. + cont, err := trie.VerifyRangeProof(root, origin, last, keys, vals, proof) + return &proofResult{ + keys: keys, + vals: vals, + diskMore: diskMore, + trieMore: cont, + proofErr: err, + tr: tr}, + nil +} + +// onStateCallback is a function that is called by generateRange, when processing a range of +// accounts or storage slots. For each element, the callback is invoked. +// +// - If 'delete' is true, then this element (and potential slots) needs to be deleted from the snapshot. +// - If 'write' is true, then this element needs to be updated with the 'val'. +// - If 'write' is false, then this element is already correct, and needs no update. +// The 'val' is the canonical encoding of the value (not the slim format for accounts) +// +// However, for accounts, the storage trie of the account needs to be checked. Also, +// dangling storages(storage exists but the corresponding account is missing) need to +// be cleaned up. +type onStateCallback func(key []byte, val []byte, write bool, delete bool) error + +// generateRange generates the state segment with particular prefix. Generation can +// either verify the correctness of existing state through range-proof and skip +// generation, or iterate trie to regenerate state on demand. +func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) { + // Use range prover to check the validity of the flat state in the range + result, err := dl.proveRange(ctx, trieId, prefix, kind, origin, max, valueConvertFn) + if err != nil { + return false, nil, err + } + last := result.last() + + // Construct contextual logger + logCtx := []interface{}{"kind", kind, "prefix", hexutil.Encode(prefix)} + if len(origin) > 0 { + logCtx = append(logCtx, "origin", hexutil.Encode(origin)) + } + logger := utils.GetLogger().New() + + // The range prover says the range is correct, skip trie iteration + if result.valid() { + snapSuccessfulRangeProofMeter.Mark(1) + logger.Trace("Proved state range", "last", hexutil.Encode(last)) + + // The verification is passed, process each state with the given + // callback function. If this state represents a contract, the + // corresponding storage check will be performed in the callback + if err := result.forEach(func(key []byte, val []byte) error { return onState(key, val, false, false) }); err != nil { + return false, nil, err + } + // Only abort the iteration when both database and trie are exhausted + return !result.diskMore && !result.trieMore, last, nil + } + logger.Trace("Detected outdated state range", "last", hexutil.Encode(last), "err", result.proofErr) + snapFailedRangeProofMeter.Mark(1) + + // Special case, the entire trie is missing. In the original trie scheme, + // all the duplicated subtries will be filtered out (only one copy of data + // will be stored). While in the snapshot model, all the storage tries + // belong to different contracts will be kept even they are duplicated. + // Track it to a certain extent remove the noise data used for statistics. + if origin == nil && last == nil { + meter := snapMissallAccountMeter + if kind == snapStorage { + meter = snapMissallStorageMeter + } + meter.Mark(1) + } + // We use the snap data to build up a cache which can be used by the + // main account trie as a primary lookup when resolving hashes + var resolver trie.NodeResolver + if len(result.keys) > 0 { + mdb := rawdb.NewMemoryDatabase() + tdb := trie.NewDatabase(mdb) + snapTrie := trie.NewEmpty(tdb) + for i, key := range result.keys { + snapTrie.Update(key, result.vals[i]) + } + root, nodes := snapTrie.Commit(false) + if nodes != nil { + tdb.Update(trie.NewWithNodeSet(nodes)) + tdb.Commit(root, false) + } + resolver = func(owner common.Hash, path []byte, hash common.Hash) []byte { + return rawdb.ReadTrieNode(mdb, owner, path, hash, tdb.Scheme()) + } + } + // Construct the trie for state iteration, reuse the trie + // if it's already opened with some nodes resolved. + tr := result.tr + if tr == nil { + tr, err = trie.New(trieId, dl.triedb) + if err != nil { + ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker) + return false, nil, errMissingTrie + } + } + var ( + trieMore bool + nodeIt = tr.NodeIterator(origin) + iter = trie.NewIterator(nodeIt) + kvkeys, kvvals = result.keys, result.vals + + // counters + count = 0 // number of states delivered by iterator + created = 0 // states created from the trie + updated = 0 // states updated from the trie + deleted = 0 // states not in trie, but were in snapshot + untouched = 0 // states already correct + + // timers + start = time.Now() + internal time.Duration + ) + nodeIt.AddResolver(resolver) + + for iter.Next() { + if last != nil && bytes.Compare(iter.Key, last) > 0 { + trieMore = true + break + } + count++ + write := true + created++ + for len(kvkeys) > 0 { + if cmp := bytes.Compare(kvkeys[0], iter.Key); cmp < 0 { + // delete the key + istart := time.Now() + if err := onState(kvkeys[0], nil, false, true); err != nil { + return false, nil, err + } + kvkeys = kvkeys[1:] + kvvals = kvvals[1:] + deleted++ + internal += time.Since(istart) + continue + } else if cmp == 0 { + // the snapshot key can be overwritten + created-- + if write = !bytes.Equal(kvvals[0], iter.Value); write { + updated++ + } else { + untouched++ + } + kvkeys = kvkeys[1:] + kvvals = kvvals[1:] + } + break + } + istart := time.Now() + if err := onState(iter.Key, iter.Value, write, false); err != nil { + return false, nil, err + } + internal += time.Since(istart) + } + if iter.Err != nil { + return false, nil, iter.Err + } + // Delete all stale snapshot states remaining + istart := time.Now() + for _, key := range kvkeys { + if err := onState(key, nil, false, true); err != nil { + return false, nil, err + } + deleted += 1 + } + internal += time.Since(istart) + + // Update metrics for counting trie iteration + if kind == snapStorage { + snapStorageTrieReadCounter.Inc((time.Since(start) - internal).Nanoseconds()) + } else { + snapAccountTrieReadCounter.Inc((time.Since(start) - internal).Nanoseconds()) + } + logger.Debug("Regenerated state range", "root", trieId.Root, "last", hexutil.Encode(last), + "count", count, "created", created, "updated", updated, "untouched", untouched, "deleted", deleted) + + // If there are either more trie items, or there are more snap items + // (in the next segment), then we need to keep working + return !trieMore && !result.diskMore, last, nil +} + +// checkAndFlush checks if an interruption signal is received or the +// batch size has exceeded the allowance. +func (dl *diskLayer) checkAndFlush(ctx *generatorContext, current []byte) error { + var abort chan *generatorStats + select { + case abort = <-dl.genAbort: + default: + } + if ctx.batch.ValueSize() > ethdb.IdealBatchSize || abort != nil { + if bytes.Compare(current, dl.genMarker) < 0 { + utils.Logger().Error(). + Str("current", fmt.Sprintf("%x", current)). + Str("genMarker", fmt.Sprintf("%x", dl.genMarker)). + Msg("Snapshot generator went backwards") + } + // Flush out the batch anyway no matter it's empty or not. + // It's possible that all the states are recovered and the + // generation indeed makes progress. + journalProgress(ctx.batch, current, ctx.stats) + + if err := ctx.batch.Write(); err != nil { + return err + } + ctx.batch.Reset() + + dl.lock.Lock() + dl.genMarker = current + dl.lock.Unlock() + + if abort != nil { + ctx.stats.Log("Aborting state snapshot generation", dl.root, current) + return newAbortErr(abort) // bubble up an error for interruption + } + // Don't hold the iterators too long, release them to let compactor works + ctx.reopenIterator(snapAccount) + ctx.reopenIterator(snapStorage) + } + if time.Since(ctx.logged) > 8*time.Second { + ctx.stats.Log("Generating state snapshot", dl.root, current) + ctx.logged = time.Now() + } + return nil +} + +// generateStorages generates the missing storage slots of the specific contract. +// It's supposed to restart the generation from the given origin position. +func generateStorages(ctx *generatorContext, dl *diskLayer, stateRoot common.Hash, account common.Hash, storageRoot common.Hash, storeMarker []byte) error { + onStorage := func(key []byte, val []byte, write bool, delete bool) error { + defer func(start time.Time) { + snapStorageWriteCounter.Inc(time.Since(start).Nanoseconds()) + }(time.Now()) + + if delete { + rawdb.DeleteStorageSnapshot(ctx.batch, account, common.BytesToHash(key)) + snapWipedStorageMeter.Mark(1) + return nil + } + if write { + rawdb.WriteStorageSnapshot(ctx.batch, account, common.BytesToHash(key), val) + snapGeneratedStorageMeter.Mark(1) + } else { + snapRecoveredStorageMeter.Mark(1) + } + ctx.stats.storage += common.StorageSize(1 + 2*common.HashLength + len(val)) + ctx.stats.slots++ + + // If we've exceeded our batch allowance or termination was requested, flush to disk + if err := dl.checkAndFlush(ctx, append(account[:], key...)); err != nil { + return err + } + return nil + } + // Loop for re-generating the missing storage slots. + var origin = common.CopyBytes(storeMarker) + for { + id := trie.StorageTrieID(stateRoot, account, storageRoot) + exhausted, last, err := dl.generateRange(ctx, id, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil) + if err != nil { + return err // The procedure it aborted, either by external signal or internal error. + } + // Abort the procedure if the entire contract storage is generated + if exhausted { + break + } + if origin = increaseKey(last); origin == nil { + break // special case, the last is 0xffffffff...fff + } + } + return nil +} + +// generateAccounts generates the missing snapshot accounts as well as their +// storage slots in the main trie. It's supposed to restart the generation +// from the given origin position. +func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) error { + onAccount := func(key []byte, val []byte, write bool, delete bool) error { + // Make sure to clear all dangling storages before this account + account := common.BytesToHash(key) + ctx.removeStorageBefore(account) + + start := time.Now() + if delete { + rawdb.DeleteAccountSnapshot(ctx.batch, account) + snapWipedAccountMeter.Mark(1) + snapAccountWriteCounter.Inc(time.Since(start).Nanoseconds()) + + ctx.removeStorageAt(account) + return nil + } + // Retrieve the current account and flatten it into the internal format + var acc struct { + Nonce uint64 + Balance *big.Int + Root common.Hash + CodeHash []byte + } + if err := rlp.DecodeBytes(val, &acc); err != nil { + utils.Logger().Fatal().Err(err).Msg("Invalid account encountered during snapshot creation") + } + // If the account is not yet in-progress, write it out + if accMarker == nil || !bytes.Equal(account[:], accMarker) { + dataLen := len(val) // Approximate size, saves us a round of RLP-encoding + if !write { + if bytes.Equal(acc.CodeHash, types.EmptyCodeHash[:]) { + dataLen -= 32 + } + if acc.Root == types.EmptyRootHash { + dataLen -= 32 + } + snapRecoveredAccountMeter.Mark(1) + } else { + data := SlimAccountRLP(acc.Nonce, acc.Balance, acc.Root, acc.CodeHash) + dataLen = len(data) + rawdb.WriteAccountSnapshot(ctx.batch, account, data) + snapGeneratedAccountMeter.Mark(1) + } + ctx.stats.storage += common.StorageSize(1 + common.HashLength + dataLen) + ctx.stats.accounts++ + } + // If the snap generation goes here after interrupted, genMarker may go backward + // when last genMarker is consisted of accountHash and storageHash + marker := account[:] + if accMarker != nil && bytes.Equal(marker, accMarker) && len(dl.genMarker) > common.HashLength { + marker = dl.genMarker[:] + } + // If we've exceeded our batch allowance or termination was requested, flush to disk + if err := dl.checkAndFlush(ctx, marker); err != nil { + return err + } + snapAccountWriteCounter.Inc(time.Since(start).Nanoseconds()) // let's count flush time as well + + // If the iterated account is the contract, create a further loop to + // verify or regenerate the contract storage. + if acc.Root == types.EmptyRootHash { + ctx.removeStorageAt(account) + } else { + var storeMarker []byte + if accMarker != nil && bytes.Equal(account[:], accMarker) && len(dl.genMarker) > common.HashLength { + storeMarker = dl.genMarker[common.HashLength:] + } + if err := generateStorages(ctx, dl, dl.root, account, acc.Root, storeMarker); err != nil { + return err + } + } + // Some account processed, unmark the marker + accMarker = nil + return nil + } + // Always reset the initial account range as 1 whenever recover from the + // interruption. TODO(rjl493456442) can we remove it? + var accountRange = accountCheckRange + if len(accMarker) > 0 { + accountRange = 1 + } + origin := common.CopyBytes(accMarker) + for { + id := trie.StateTrieID(dl.root) + exhausted, last, err := dl.generateRange(ctx, id, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP) + if err != nil { + return err // The procedure it aborted, either by external signal or internal error. + } + origin = increaseKey(last) + + // Last step, cleanup the storages after the last account. + // All the left storages should be treated as dangling. + if origin == nil || exhausted { + ctx.removeStorageLeft() + break + } + accountRange = accountCheckRange + } + return nil +} + +// generate is a background thread that iterates over the state and storage tries, +// constructing the state snapshot. All the arguments are purely for statistics +// gathering and logging, since the method surfs the blocks as they arrive, often +// being restarted. +func (dl *diskLayer) generate(stats *generatorStats) { + var ( + accMarker []byte + abort chan *generatorStats + ) + if len(dl.genMarker) > 0 { // []byte{} is the start, use nil for that + accMarker = dl.genMarker[:common.HashLength] + } + stats.Log("Resuming state snapshot generation", dl.root, dl.genMarker) + + // Initialize the global generator context. The snapshot iterators are + // opened at the interrupted position because the assumption is held + // that all the snapshot data are generated correctly before the marker. + // Even if the snapshot data is updated during the interruption (before + // or at the marker), the assumption is still held. + // For the account or storage slot at the interruption, they will be + // processed twice by the generator(they are already processed in the + // last run) but it's fine. + ctx := newGeneratorContext(stats, dl.diskdb, accMarker, dl.genMarker) + defer ctx.close() + + if err := generateAccounts(ctx, dl, accMarker); err != nil { + // Extract the received interruption signal if exists + if aerr, ok := err.(*abortErr); ok { + abort = aerr.abort + } + // Aborted by internal error, wait the signal + if abort == nil { + abort = <-dl.genAbort + } + abort <- stats + return + } + // Snapshot fully generated, set the marker to nil. + // Note even there is nothing to commit, persist the + // generator anyway to mark the snapshot is complete. + journalProgress(ctx.batch, nil, stats) + if err := ctx.batch.Write(); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to flush batch") + + abort = <-dl.genAbort + abort <- stats + return + } + ctx.batch.Reset() + + utils.Logger().Info(). + Uint64("accounts", stats.accounts). + Uint64("slots", stats.slots). + Interface("storage", stats.storage). + Uint64("dangling", stats.dangling). + Interface("elapsed", common.PrettyDuration(time.Since(stats.start))). + Msg("Generated state snapshot") + + dl.lock.Lock() + dl.genMarker = nil + close(dl.genPending) + dl.lock.Unlock() + + // Someone will be looking for us, wait it out + abort = <-dl.genAbort + abort <- nil +} + +// increaseKey increase the input key by one bit. Return nil if the entire +// addition operation overflows. +func increaseKey(key []byte) []byte { + for i := len(key) - 1; i >= 0; i-- { + key[i]++ + if key[i] != 0x0 { + return key + } + } + return nil +} + +// abortErr wraps an interruption signal received to represent the +// generation is aborted by external processes. +type abortErr struct { + abort chan *generatorStats +} + +func newAbortErr(abort chan *generatorStats) error { + return &abortErr{abort: abort} +} + +func (err *abortErr) Error() string { + return "aborted" +} diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go new file mode 100644 index 0000000000..c5a6725f40 --- /dev/null +++ b/core/state/snapshot/generate_test.go @@ -0,0 +1,861 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "fmt" + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/harmony-one/harmony/core/rawdb" + "golang.org/x/crypto/sha3" +) + +func hashData(input []byte) common.Hash { + var hasher = sha3.NewLegacyKeccak256() + var hash common.Hash + hasher.Reset() + hasher.Write(input) + hasher.Sum(hash[:0]) + return hash +} + +// Tests that snapshot generation from an empty database. +func TestGeneration(t *testing.T) { + // We can't use statedb to make a test trie (circular dependency), so make + // a fake one manually. We're going with a small account trie of 3 accounts, + // two of which also has the same 3-slot storage trie attached. + var helper = newHelper() + stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false) + + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + + root, snap := helper.CommitAndGenerate() + if have, want := root, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"); have != want { + t.Fatalf("have %#x want %#x", have, want) + } + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(3 * time.Second): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + +// Tests that snapshot generation with existent flat state. +func TestGenerateExistentState(t *testing.T) { + // We can't use statedb to make a test trie (circular dependency), so make + // a fake one manually. We're going with a small account trie of 3 accounts, + // two of which also has the same 3-slot storage trie attached. + var helper = newHelper() + + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) + + stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + root, snap := helper.CommitAndGenerate() + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(3 * time.Second): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + +func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) { + t.Helper() + + accIt := snap.AccountIterator(common.Hash{}) + defer accIt.Release() + + snapRoot, err := generateTrieRoot(nil, "", accIt, common.Hash{}, stackTrieGenerate, + func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) { + storageIt, _ := snap.StorageIterator(accountHash, common.Hash{}) + defer storageIt.Release() + + hash, err := generateTrieRoot(nil, "", storageIt, accountHash, stackTrieGenerate, nil, stat, false) + if err != nil { + return common.Hash{}, err + } + return hash, nil + }, newGenerateStats(), true) + if err != nil { + t.Fatal(err) + } + if snapRoot != trieRoot { + t.Fatalf("snaproot: %#x != trieroot #%x", snapRoot, trieRoot) + } + if err := CheckDanglingStorage(snap.diskdb); err != nil { + t.Fatalf("Detected dangling storages: %v", err) + } +} + +type testHelper struct { + diskdb ethdb.Database + triedb *trie.Database + accTrie *trie.StateTrie + nodes *trie.MergedNodeSet +} + +func newHelper() *testHelper { + diskdb := rawdb.NewMemoryDatabase() + triedb := trie.NewDatabase(diskdb) + accTrie, _ := trie.NewStateTrie(trie.StateTrieID(common.Hash{}), triedb) + return &testHelper{ + diskdb: diskdb, + triedb: triedb, + accTrie: accTrie, + nodes: trie.NewMergedNodeSet(), + } +} + +func (t *testHelper) addTrieAccount(acckey string, acc *Account) { + val, _ := rlp.EncodeToBytes(acc) + t.accTrie.Update([]byte(acckey), val) +} + +func (t *testHelper) addSnapAccount(acckey string, acc *Account) { + val, _ := rlp.EncodeToBytes(acc) + key := hashData([]byte(acckey)) + rawdb.WriteAccountSnapshot(t.diskdb, key, val) +} + +func (t *testHelper) addAccount(acckey string, acc *Account) { + t.addTrieAccount(acckey, acc) + t.addSnapAccount(acckey, acc) +} + +func (t *testHelper) addSnapStorage(accKey string, keys []string, vals []string) { + accHash := hashData([]byte(accKey)) + for i, key := range keys { + rawdb.WriteStorageSnapshot(t.diskdb, accHash, hashData([]byte(key)), []byte(vals[i])) + } +} + +func (t *testHelper) makeStorageTrie(stateRoot, owner common.Hash, keys []string, vals []string, commit bool) []byte { + id := trie.StorageTrieID(stateRoot, owner, common.Hash{}) + stTrie, _ := trie.NewStateTrie(id, t.triedb) + for i, k := range keys { + stTrie.Update([]byte(k), []byte(vals[i])) + } + if !commit { + return stTrie.Hash().Bytes() + } + root, nodes := stTrie.Commit(false) + if nodes != nil { + t.nodes.Merge(nodes) + } + return root.Bytes() +} + +func (t *testHelper) Commit() common.Hash { + root, nodes := t.accTrie.Commit(true) + if nodes != nil { + t.nodes.Merge(nodes) + } + t.triedb.Update(t.nodes) + t.triedb.Commit(root, false) + return root +} + +func (t *testHelper) CommitAndGenerate() (common.Hash, *diskLayer) { + root := t.Commit() + snap := generateSnapshot(t.diskdb, t.triedb, 16, root) + return root, snap +} + +// Tests that snapshot generation with existent flat state, where the flat state +// contains some errors: +// - the contract with empty storage root but has storage entries in the disk +// - the contract with non empty storage root but empty storage slots +// - the contract(non-empty storage) misses some storage slots +// - miss in the beginning +// - miss in the middle +// - miss in the end +// +// - the contract(non-empty storage) has wrong storage slots +// - wrong slots in the beginning +// - wrong slots in the middle +// - wrong slots in the end +// +// - the contract(non-empty storage) has extra storage slots +// - extra slots in the beginning +// - extra slots in the middle +// - extra slots in the end +func TestGenerateExistentStateWithWrongStorage(t *testing.T) { + helper := newHelper() + + // Account one, empty root but non-empty database + helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + // Account two, non empty root but empty database + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + + // Miss slots + { + // Account three, non empty root but misses slots in the beginning + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapStorage("acc-3", []string{"key-2", "key-3"}, []string{"val-2", "val-3"}) + + // Account four, non empty root but misses slots in the middle + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapStorage("acc-4", []string{"key-1", "key-3"}, []string{"val-1", "val-3"}) + + // Account five, non empty root but misses slots in the end + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-5")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addAccount("acc-5", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapStorage("acc-5", []string{"key-1", "key-2"}, []string{"val-1", "val-2"}) + } + + // Wrong storage slots + { + // Account six, non empty root but wrong slots in the beginning + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapStorage("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"badval-1", "val-2", "val-3"}) + + // Account seven, non empty root but wrong slots in the middle + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-7")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addAccount("acc-7", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapStorage("acc-7", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "badval-2", "val-3"}) + + // Account eight, non empty root but wrong slots in the end + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-8")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addAccount("acc-8", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapStorage("acc-8", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "badval-3"}) + + // Account 9, non empty root but rotated slots + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-9")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addAccount("acc-9", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapStorage("acc-9", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-3", "val-2"}) + } + + // Extra storage slots + { + // Account 10, non empty root but extra slots in the beginning + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-10")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addAccount("acc-10", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapStorage("acc-10", []string{"key-0", "key-1", "key-2", "key-3"}, []string{"val-0", "val-1", "val-2", "val-3"}) + + // Account 11, non empty root but extra slots in the middle + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-11")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addAccount("acc-11", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapStorage("acc-11", []string{"key-1", "key-2", "key-2-1", "key-3"}, []string{"val-1", "val-2", "val-2-1", "val-3"}) + + // Account 12, non empty root but extra slots in the end + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-12")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addAccount("acc-12", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapStorage("acc-12", []string{"key-1", "key-2", "key-3", "key-4"}, []string{"val-1", "val-2", "val-3", "val-4"}) + } + + root, snap := helper.CommitAndGenerate() + t.Logf("Root: %#x\n", root) // Root = 0x8746cce9fd9c658b2cfd639878ed6584b7a2b3e73bb40f607fcfa156002429a0 + + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(3 * time.Second): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + +// Tests that snapshot generation with existent flat state, where the flat state +// contains some errors: +// - miss accounts +// - wrong accounts +// - extra accounts +func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { + helper := newHelper() + + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + + // Trie accounts [acc-1, acc-2, acc-3, acc-4, acc-6] + // Extra accounts [acc-0, acc-5, acc-7] + + // Missing accounts, only in the trie + { + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Beginning + helper.addTrieAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Middle + helper.addTrieAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // End + } + + // Wrong accounts + { + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: common.Hex2Bytes("0x1234")}) + + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-3", &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) + } + + // Extra accounts, only in the snap + { + helper.addSnapAccount("acc-0", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // before the beginning + helper.addSnapAccount("acc-5", &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: common.Hex2Bytes("0x1234")}) // Middle + helper.addSnapAccount("acc-7", &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // after the end + } + + root, snap := helper.CommitAndGenerate() + t.Logf("Root: %#x\n", root) // Root = 0x825891472281463511e7ebcc7f109e4f9200c20fa384754e11fd605cd98464e8 + + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(3 * time.Second): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + +// Tests that snapshot generation errors out correctly in case of a missing trie +// node in the account trie. +func TestGenerateCorruptAccountTrie(t *testing.T) { + // We can't use statedb to make a test trie (circular dependency), so make + // a fake one manually. We're going with a small account trie of 3 accounts, + // without any storage slots to keep the test smaller. + helper := newHelper() + + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4 + + root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 + + // Delete an account trie leaf and ensure the generator chokes + helper.triedb.Commit(root, false) + helper.diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes()) + + snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) + select { + case <-snap.genPending: + // Snapshot generation succeeded + t.Errorf("Snapshot generated against corrupt account trie") + + case <-time.After(time.Second): + // Not generated fast enough, hopefully blocked inside on missing trie node fail + } + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + +// Tests that snapshot generation errors out correctly in case of a missing root +// trie node for a storage trie. It's similar to internal corruption but it is +// handled differently inside the generator. +func TestGenerateMissingStorageTrie(t *testing.T) { + // We can't use statedb to make a test trie (circular dependency), so make + // a fake one manually. We're going with a small account trie of 3 accounts, + // two of which also has the same 3-slot storage trie attached. + helper := newHelper() + + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 + + root := helper.Commit() + + // Delete a storage trie root and ensure the generator chokes + helper.diskdb.Delete(stRoot) + + snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) + select { + case <-snap.genPending: + // Snapshot generation succeeded + t.Errorf("Snapshot generated against corrupt storage trie") + + case <-time.After(time.Second): + // Not generated fast enough, hopefully blocked inside on missing trie node fail + } + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + +// Tests that snapshot generation errors out correctly in case of a missing trie +// node in a storage trie. +func TestGenerateCorruptStorageTrie(t *testing.T) { + // We can't use statedb to make a test trie (circular dependency), so make + // a fake one manually. We're going with a small account trie of 3 accounts, + // two of which also has the same 3-slot storage trie attached. + helper := newHelper() + + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 + + root := helper.Commit() + + // Delete a storage trie leaf and ensure the generator chokes + helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes()) + + snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) + select { + case <-snap.genPending: + // Snapshot generation succeeded + t.Errorf("Snapshot generated against corrupt storage trie") + + case <-time.After(time.Second): + // Not generated fast enough, hopefully blocked inside on missing trie node fail + } + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + +// Tests that snapshot generation when an extra account with storage exists in the snap state. +func TestGenerateWithExtraAccounts(t *testing.T) { + helper := newHelper() + { + // Account one in the trie + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), + []string{"key-1", "key-2", "key-3", "key-4", "key-5"}, + []string{"val-1", "val-2", "val-3", "val-4", "val-5"}, + true, + ) + acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} + val, _ := rlp.EncodeToBytes(acc) + helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + + // Identical in the snap + key := hashData([]byte("acc-1")) + rawdb.WriteAccountSnapshot(helper.diskdb, key, val) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-1")), []byte("val-1")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-2")), []byte("val-2")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-3")), []byte("val-3")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-4")), []byte("val-4")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-5")), []byte("val-5")) + } + { + // Account two exists only in the snapshot + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), + []string{"key-1", "key-2", "key-3", "key-4", "key-5"}, + []string{"val-1", "val-2", "val-3", "val-4", "val-5"}, + true, + ) + acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} + val, _ := rlp.EncodeToBytes(acc) + key := hashData([]byte("acc-2")) + rawdb.WriteAccountSnapshot(helper.diskdb, key, val) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("b-key-1")), []byte("b-val-1")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("b-key-2")), []byte("b-val-2")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("b-key-3")), []byte("b-val-3")) + } + root := helper.Commit() + + // To verify the test: If we now inspect the snap db, there should exist extraneous storage items + if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil { + t.Fatalf("expected snap storage to exist") + } + snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(3 * time.Second): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop + // If we now inspect the snap db, there should exist no extraneous storage items + if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { + t.Fatalf("expected slot to be removed, got %v", string(data)) + } +} + +// Tests that snapshot generation when an extra account with storage exists in the snap state. +func TestGenerateWithManyExtraAccounts(t *testing.T) { + helper := newHelper() + { + // Account one in the trie + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), + []string{"key-1", "key-2", "key-3"}, + []string{"val-1", "val-2", "val-3"}, + true, + ) + acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} + val, _ := rlp.EncodeToBytes(acc) + helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + + // Identical in the snap + key := hashData([]byte("acc-1")) + rawdb.WriteAccountSnapshot(helper.diskdb, key, val) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-1")), []byte("val-1")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-2")), []byte("val-2")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-3")), []byte("val-3")) + } + { + // 100 accounts exist only in snapshot + for i := 0; i < 1000; i++ { + acc := &Account{Balance: big.NewInt(int64(i)), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()} + val, _ := rlp.EncodeToBytes(acc) + key := hashData([]byte(fmt.Sprintf("acc-%d", i))) + rawdb.WriteAccountSnapshot(helper.diskdb, key, val) + } + } + root, snap := helper.CommitAndGenerate() + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(3 * time.Second): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + +// Tests this case +// maxAccountRange 3 +// snapshot-accounts: 01, 02, 03, 04, 05, 06, 07 +// trie-accounts: 03, 07 +// +// We iterate three snapshot storage slots (max = 3) from the database. They are 0x01, 0x02, 0x03. +// The trie has a lot of deletions. +// So in trie, we iterate 2 entries 0x03, 0x07. We create the 0x07 in the database and abort the procedure, because the trie is exhausted. +// But in the database, we still have the stale storage slots 0x04, 0x05. They are not iterated yet, but the procedure is finished. +func TestGenerateWithExtraBeforeAndAfter(t *testing.T) { + accountCheckRange = 3 + helper := newHelper() + { + acc := &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()} + val, _ := rlp.EncodeToBytes(acc) + helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val) + helper.accTrie.Update(common.HexToHash("0x07").Bytes(), val) + + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x01"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x02"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x03"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x04"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x05"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x06"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x07"), val) + } + root, snap := helper.CommitAndGenerate() + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(3 * time.Second): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + +// TestGenerateWithMalformedSnapdata tests what happes if we have some junk +// in the snapshot database, which cannot be parsed back to an account +func TestGenerateWithMalformedSnapdata(t *testing.T) { + accountCheckRange = 3 + helper := newHelper() + { + acc := &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()} + val, _ := rlp.EncodeToBytes(acc) + helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val) + + junk := make([]byte, 100) + copy(junk, []byte{0xde, 0xad}) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x02"), junk) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x03"), junk) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x04"), junk) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x05"), junk) + } + root, snap := helper.CommitAndGenerate() + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(3 * time.Second): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop + // If we now inspect the snap db, there should exist no extraneous storage items + if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { + t.Fatalf("expected slot to be removed, got %v", string(data)) + } +} + +func TestGenerateFromEmptySnap(t *testing.T) { + //enableLogging() + accountCheckRange = 10 + storageCheckRange = 20 + helper := newHelper() + // Add 1K accounts to the trie + for i := 0; i < 400; i++ { + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount(fmt.Sprintf("acc-%d", i), + &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + } + root, snap := helper.CommitAndGenerate() + t.Logf("Root: %#x\n", root) // Root: 0x6f7af6d2e1a1bf2b84a3beb3f8b64388465fbc1e274ca5d5d3fc787ca78f59e4 + + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(3 * time.Second): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + +// Tests that snapshot generation with existent flat state, where the flat state +// storage is correct, but incomplete. +// The incomplete part is on the second range +// snap: [ 0x01, 0x02, 0x03, 0x04] , [ 0x05, 0x06, 0x07, {missing}] (with storageCheck = 4) +// trie: 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 +// This hits a case where the snap verification passes, but there are more elements in the trie +// which we must also add. +func TestGenerateWithIncompleteStorage(t *testing.T) { + storageCheckRange = 4 + helper := newHelper() + stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"} + stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"} + // We add 8 accounts, each one is missing exactly one of the storage slots. This means + // we don't have to order the keys and figure out exactly which hash-key winds up + // on the sensitive spots at the boundaries + for i := 0; i < 8; i++ { + accKey := fmt.Sprintf("acc-%d", i) + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(accKey)), stKeys, stVals, true) + helper.addAccount(accKey, &Account{Balance: big.NewInt(int64(i)), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + var moddedKeys []string + var moddedVals []string + for ii := 0; ii < 8; ii++ { + if ii != i { + moddedKeys = append(moddedKeys, stKeys[ii]) + moddedVals = append(moddedVals, stVals[ii]) + } + } + helper.addSnapStorage(accKey, moddedKeys, moddedVals) + } + root, snap := helper.CommitAndGenerate() + t.Logf("Root: %#x\n", root) // Root: 0xca73f6f05ba4ca3024ef340ef3dfca8fdabc1b677ff13f5a9571fd49c16e67ff + + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(3 * time.Second): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + +func incKey(key []byte) []byte { + for i := len(key) - 1; i >= 0; i-- { + key[i]++ + if key[i] != 0x0 { + break + } + } + return key +} + +func decKey(key []byte) []byte { + for i := len(key) - 1; i >= 0; i-- { + key[i]-- + if key[i] != 0xff { + break + } + } + return key +} + +func populateDangling(disk ethdb.KeyValueStore) { + populate := func(accountHash common.Hash, keys []string, vals []string) { + for i, key := range keys { + rawdb.WriteStorageSnapshot(disk, accountHash, hashData([]byte(key)), []byte(vals[i])) + } + } + // Dangling storages of the "first" account + populate(common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + // Dangling storages of the "last" account + populate(common.HexToHash("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + // Dangling storages around the account 1 + hash := decKey(hashData([]byte("acc-1")).Bytes()) + populate(common.BytesToHash(hash), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + hash = incKey(hashData([]byte("acc-1")).Bytes()) + populate(common.BytesToHash(hash), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + // Dangling storages around the account 2 + hash = decKey(hashData([]byte("acc-2")).Bytes()) + populate(common.BytesToHash(hash), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + hash = incKey(hashData([]byte("acc-2")).Bytes()) + populate(common.BytesToHash(hash), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + // Dangling storages around the account 3 + hash = decKey(hashData([]byte("acc-3")).Bytes()) + populate(common.BytesToHash(hash), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + hash = incKey(hashData([]byte("acc-3")).Bytes()) + populate(common.BytesToHash(hash), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + // Dangling storages of the random account + populate(randomHash(), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + populate(randomHash(), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + populate(randomHash(), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) +} + +// Tests that snapshot generation with dangling storages. Dangling storage means +// the storage data is existent while the corresponding account data is missing. +// +// This test will populate some dangling storages to see if they can be cleaned up. +func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { + var helper = newHelper() + + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) + + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + + helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + populateDangling(helper.diskdb) + + root, snap := helper.CommitAndGenerate() + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(3 * time.Second): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} + +// Tests that snapshot generation with dangling storages. Dangling storage means +// the storage data is existent while the corresponding account data is missing. +// +// This test will populate some dangling storages to see if they can be cleaned up. +func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) { + var helper = newHelper() + + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) + + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + + populateDangling(helper.diskdb) + + root, snap := helper.CommitAndGenerate() + select { + case <-snap.genPending: + // Snapshot generation succeeded + + case <-time.After(3 * time.Second): + t.Errorf("Snapshot generation failed") + } + checkSnapRoot(t, snap, root) + + // Signal abortion to the generator and wait for it to tear down + stop := make(chan *generatorStats) + snap.genAbort <- stop + <-stop +} diff --git a/core/state/snapshot/holdable_iterator.go b/core/state/snapshot/holdable_iterator.go new file mode 100644 index 0000000000..1e86ff9d82 --- /dev/null +++ b/core/state/snapshot/holdable_iterator.go @@ -0,0 +1,97 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" +) + +// holdableIterator is a wrapper of underlying database iterator. It extends +// the basic iterator interface by adding Hold which can hold the element +// locally where the iterator is currently located and serve it up next time. +type holdableIterator struct { + it ethdb.Iterator + key []byte + val []byte + atHeld bool +} + +// newHoldableIterator initializes the holdableIterator with the given iterator. +func newHoldableIterator(it ethdb.Iterator) *holdableIterator { + return &holdableIterator{it: it} +} + +// Hold holds the element locally where the iterator is currently located which +// can be served up next time. +func (it *holdableIterator) Hold() { + if it.it.Key() == nil { + return // nothing to hold + } + it.key = common.CopyBytes(it.it.Key()) + it.val = common.CopyBytes(it.it.Value()) + it.atHeld = false +} + +// Next moves the iterator to the next key/value pair. It returns whether the +// iterator is exhausted. +func (it *holdableIterator) Next() bool { + if !it.atHeld && it.key != nil { + it.atHeld = true + } else if it.atHeld { + it.atHeld = false + it.key = nil + it.val = nil + } + if it.key != nil { + return true // shifted to locally held value + } + return it.it.Next() +} + +// Error returns any accumulated error. Exhausting all the key/value pairs +// is not considered to be an error. +func (it *holdableIterator) Error() error { return it.it.Error() } + +// Release releases associated resources. Release should always succeed and can +// be called multiple times without causing error. +func (it *holdableIterator) Release() { + it.atHeld = false + it.key = nil + it.val = nil + it.it.Release() +} + +// Key returns the key of the current key/value pair, or nil if done. The caller +// should not modify the contents of the returned slice, and its contents may +// change on the next call to Next. +func (it *holdableIterator) Key() []byte { + if it.key != nil { + return it.key + } + return it.it.Key() +} + +// Value returns the value of the current key/value pair, or nil if done. The +// caller should not modify the contents of the returned slice, and its contents +// may change on the next call to Next. +func (it *holdableIterator) Value() []byte { + if it.val != nil { + return it.val + } + return it.it.Value() +} diff --git a/core/state/snapshot/holdable_iterator_test.go b/core/state/snapshot/holdable_iterator_test.go new file mode 100644 index 0000000000..76a2c75c0c --- /dev/null +++ b/core/state/snapshot/holdable_iterator_test.go @@ -0,0 +1,163 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "bytes" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/harmony/core/rawdb" +) + +func TestIteratorHold(t *testing.T) { + // Create the key-value data store + var ( + content = map[string]string{"k1": "v1", "k2": "v2", "k3": "v3"} + order = []string{"k1", "k2", "k3"} + db = rawdb.NewMemoryDatabase() + ) + for key, val := range content { + if err := db.Put([]byte(key), []byte(val)); err != nil { + t.Fatalf("failed to insert item %s:%s into database: %v", key, val, err) + } + } + // Iterate over the database with the given configs and verify the results + it, idx := newHoldableIterator(db.NewIterator(nil, nil)), 0 + + // Nothing should be affected for calling Discard on non-initialized iterator + it.Hold() + + for it.Next() { + if len(content) <= idx { + t.Errorf("more items than expected: checking idx=%d (key %q), expecting len=%d", idx, it.Key(), len(order)) + break + } + if !bytes.Equal(it.Key(), []byte(order[idx])) { + t.Errorf("item %d: key mismatch: have %s, want %s", idx, string(it.Key()), order[idx]) + } + if !bytes.Equal(it.Value(), []byte(content[order[idx]])) { + t.Errorf("item %d: value mismatch: have %s, want %s", idx, string(it.Value()), content[order[idx]]) + } + // Should be safe to call discard multiple times + it.Hold() + it.Hold() + + // Shift iterator to the discarded element + it.Next() + if !bytes.Equal(it.Key(), []byte(order[idx])) { + t.Errorf("item %d: key mismatch: have %s, want %s", idx, string(it.Key()), order[idx]) + } + if !bytes.Equal(it.Value(), []byte(content[order[idx]])) { + t.Errorf("item %d: value mismatch: have %s, want %s", idx, string(it.Value()), content[order[idx]]) + } + + // Discard/Next combo should work always + it.Hold() + it.Next() + if !bytes.Equal(it.Key(), []byte(order[idx])) { + t.Errorf("item %d: key mismatch: have %s, want %s", idx, string(it.Key()), order[idx]) + } + if !bytes.Equal(it.Value(), []byte(content[order[idx]])) { + t.Errorf("item %d: value mismatch: have %s, want %s", idx, string(it.Value()), content[order[idx]]) + } + idx++ + } + if err := it.Error(); err != nil { + t.Errorf("iteration failed: %v", err) + } + if idx != len(order) { + t.Errorf("iteration terminated prematurely: have %d, want %d", idx, len(order)) + } + db.Close() +} + +func TestReopenIterator(t *testing.T) { + var ( + content = map[common.Hash]string{ + common.HexToHash("a1"): "v1", + common.HexToHash("a2"): "v2", + common.HexToHash("a3"): "v3", + common.HexToHash("a4"): "v4", + common.HexToHash("a5"): "v5", + common.HexToHash("a6"): "v6", + } + order = []common.Hash{ + common.HexToHash("a1"), + common.HexToHash("a2"), + common.HexToHash("a3"), + common.HexToHash("a4"), + common.HexToHash("a5"), + common.HexToHash("a6"), + } + db = rawdb.NewMemoryDatabase() + ) + for key, val := range content { + rawdb.WriteAccountSnapshot(db, key, []byte(val)) + } + checkVal := func(it *holdableIterator, index int) { + if !bytes.Equal(it.Key(), append(rawdb.SnapshotAccountPrefix, order[index].Bytes()...)) { + t.Fatalf("Unexpected data entry key, want %v got %v", order[index], it.Key()) + } + if !bytes.Equal(it.Value(), []byte(content[order[index]])) { + t.Fatalf("Unexpected data entry key, want %v got %v", []byte(content[order[index]]), it.Value()) + } + } + // Iterate over the database with the given configs and verify the results + ctx, idx := newGeneratorContext(&generatorStats{}, db, nil, nil), -1 + + idx++ + ctx.account.Next() + checkVal(ctx.account, idx) + + ctx.reopenIterator(snapAccount) + idx++ + ctx.account.Next() + checkVal(ctx.account, idx) + + // reopen twice + ctx.reopenIterator(snapAccount) + ctx.reopenIterator(snapAccount) + idx++ + ctx.account.Next() + checkVal(ctx.account, idx) + + // reopen iterator with held value + ctx.account.Next() + ctx.account.Hold() + ctx.reopenIterator(snapAccount) + idx++ + ctx.account.Next() + checkVal(ctx.account, idx) + + // reopen twice iterator with held value + ctx.account.Next() + ctx.account.Hold() + ctx.reopenIterator(snapAccount) + ctx.reopenIterator(snapAccount) + idx++ + ctx.account.Next() + checkVal(ctx.account, idx) + + // shift to the end and reopen + ctx.account.Next() // the end + ctx.reopenIterator(snapAccount) + ctx.account.Next() + if ctx.account.Key() != nil { + t.Fatal("Unexpected iterated entry") + } +} diff --git a/core/state/snapshot/iterator.go b/core/state/snapshot/iterator.go new file mode 100644 index 0000000000..44127ae6fd --- /dev/null +++ b/core/state/snapshot/iterator.go @@ -0,0 +1,400 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "bytes" + "fmt" + "sort" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/harmony-one/harmony/core/rawdb" +) + +// Iterator is an iterator to step over all the accounts or the specific +// storage in a snapshot which may or may not be composed of multiple layers. +type Iterator interface { + // Next steps the iterator forward one element, returning false if exhausted, + // or an error if iteration failed for some reason (e.g. root being iterated + // becomes stale and garbage collected). + Next() bool + + // Error returns any failure that occurred during iteration, which might have + // caused a premature iteration exit (e.g. snapshot stack becoming stale). + Error() error + + // Hash returns the hash of the account or storage slot the iterator is + // currently at. + Hash() common.Hash + + // Release releases associated resources. Release should always succeed and + // can be called multiple times without causing error. + Release() +} + +// AccountIterator is an iterator to step over all the accounts in a snapshot, +// which may or may not be composed of multiple layers. +type AccountIterator interface { + Iterator + + // Account returns the RLP encoded slim account the iterator is currently at. + // An error will be returned if the iterator becomes invalid + Account() []byte +} + +// StorageIterator is an iterator to step over the specific storage in a snapshot, +// which may or may not be composed of multiple layers. +type StorageIterator interface { + Iterator + + // Slot returns the storage slot the iterator is currently at. An error will + // be returned if the iterator becomes invalid + Slot() []byte +} + +// diffAccountIterator is an account iterator that steps over the accounts (both +// live and deleted) contained within a single diff layer. Higher order iterators +// will use the deleted accounts to skip deeper iterators. +type diffAccountIterator struct { + // curHash is the current hash the iterator is positioned on. The field is + // explicitly tracked since the referenced diff layer might go stale after + // the iterator was positioned and we don't want to fail accessing the old + // hash as long as the iterator is not touched any more. + curHash common.Hash + + layer *diffLayer // Live layer to retrieve values from + keys []common.Hash // Keys left in the layer to iterate + fail error // Any failures encountered (stale) +} + +// AccountIterator creates an account iterator over a single diff layer. +func (dl *diffLayer) AccountIterator(seek common.Hash) AccountIterator { + // Seek out the requested starting account + hashes := dl.AccountList() + index := sort.Search(len(hashes), func(i int) bool { + return bytes.Compare(seek[:], hashes[i][:]) <= 0 + }) + // Assemble and returned the already seeked iterator + return &diffAccountIterator{ + layer: dl, + keys: hashes[index:], + } +} + +// Next steps the iterator forward one element, returning false if exhausted. +func (it *diffAccountIterator) Next() bool { + // If the iterator was already stale, consider it a programmer error. Although + // we could just return false here, triggering this path would probably mean + // somebody forgot to check for Error, so lets blow up instead of undefined + // behavior that's hard to debug. + if it.fail != nil { + panic(fmt.Sprintf("called Next of failed iterator: %v", it.fail)) + } + // Stop iterating if all keys were exhausted + if len(it.keys) == 0 { + return false + } + if it.layer.Stale() { + it.fail, it.keys = ErrSnapshotStale, nil + return false + } + // Iterator seems to be still alive, retrieve and cache the live hash + it.curHash = it.keys[0] + // key cached, shift the iterator and notify the user of success + it.keys = it.keys[1:] + return true +} + +// Error returns any failure that occurred during iteration, which might have +// caused a premature iteration exit (e.g. snapshot stack becoming stale). +func (it *diffAccountIterator) Error() error { + return it.fail +} + +// Hash returns the hash of the account the iterator is currently at. +func (it *diffAccountIterator) Hash() common.Hash { + return it.curHash +} + +// Account returns the RLP encoded slim account the iterator is currently at. +// This method may _fail_, if the underlying layer has been flattened between +// the call to Next and Account. That type of error will set it.Err. +// This method assumes that flattening does not delete elements from +// the accountdata mapping (writing nil into it is fine though), and will panic +// if elements have been deleted. +// +// Note the returned account is not a copy, please don't modify it. +func (it *diffAccountIterator) Account() []byte { + it.layer.lock.RLock() + blob, ok := it.layer.accountData[it.curHash] + if !ok { + if _, ok := it.layer.destructSet[it.curHash]; ok { + it.layer.lock.RUnlock() + return nil + } + panic(fmt.Sprintf("iterator referenced non-existent account: %x", it.curHash)) + } + it.layer.lock.RUnlock() + if it.layer.Stale() { + it.fail, it.keys = ErrSnapshotStale, nil + } + return blob +} + +// Release is a noop for diff account iterators as there are no held resources. +func (it *diffAccountIterator) Release() {} + +// diskAccountIterator is an account iterator that steps over the live accounts +// contained within a disk layer. +type diskAccountIterator struct { + layer *diskLayer + it ethdb.Iterator +} + +// AccountIterator creates an account iterator over a disk layer. +func (dl *diskLayer) AccountIterator(seek common.Hash) AccountIterator { + pos := common.TrimRightZeroes(seek[:]) + return &diskAccountIterator{ + layer: dl, + it: dl.diskdb.NewIterator(rawdb.SnapshotAccountPrefix, pos), + } +} + +// Next steps the iterator forward one element, returning false if exhausted. +func (it *diskAccountIterator) Next() bool { + // If the iterator was already exhausted, don't bother + if it.it == nil { + return false + } + // Try to advance the iterator and release it if we reached the end + for { + if !it.it.Next() { + it.it.Release() + it.it = nil + return false + } + if len(it.it.Key()) == len(rawdb.SnapshotAccountPrefix)+common.HashLength { + break + } + } + return true +} + +// Error returns any failure that occurred during iteration, which might have +// caused a premature iteration exit (e.g. snapshot stack becoming stale). +// +// A diff layer is immutable after creation content wise and can always be fully +// iterated without error, so this method always returns nil. +func (it *diskAccountIterator) Error() error { + if it.it == nil { + return nil // Iterator is exhausted and released + } + return it.it.Error() +} + +// Hash returns the hash of the account the iterator is currently at. +func (it *diskAccountIterator) Hash() common.Hash { + return common.BytesToHash(it.it.Key()) // The prefix will be truncated +} + +// Account returns the RLP encoded slim account the iterator is currently at. +func (it *diskAccountIterator) Account() []byte { + return it.it.Value() +} + +// Release releases the database snapshot held during iteration. +func (it *diskAccountIterator) Release() { + // The iterator is auto-released on exhaustion, so make sure it's still alive + if it.it != nil { + it.it.Release() + it.it = nil + } +} + +// diffStorageIterator is a storage iterator that steps over the specific storage +// (both live and deleted) contained within a single diff layer. Higher order +// iterators will use the deleted slot to skip deeper iterators. +type diffStorageIterator struct { + // curHash is the current hash the iterator is positioned on. The field is + // explicitly tracked since the referenced diff layer might go stale after + // the iterator was positioned and we don't want to fail accessing the old + // hash as long as the iterator is not touched any more. + curHash common.Hash + account common.Hash + + layer *diffLayer // Live layer to retrieve values from + keys []common.Hash // Keys left in the layer to iterate + fail error // Any failures encountered (stale) +} + +// StorageIterator creates a storage iterator over a single diff layer. +// Except the storage iterator is returned, there is an additional flag +// "destructed" returned. If it's true then it means the whole storage is +// destructed in this layer(maybe recreated too), don't bother deeper layer +// for storage retrieval. +func (dl *diffLayer) StorageIterator(account common.Hash, seek common.Hash) (StorageIterator, bool) { + // Create the storage for this account even it's marked + // as destructed. The iterator is for the new one which + // just has the same address as the deleted one. + hashes, destructed := dl.StorageList(account) + index := sort.Search(len(hashes), func(i int) bool { + return bytes.Compare(seek[:], hashes[i][:]) <= 0 + }) + // Assemble and returned the already seeked iterator + return &diffStorageIterator{ + layer: dl, + account: account, + keys: hashes[index:], + }, destructed +} + +// Next steps the iterator forward one element, returning false if exhausted. +func (it *diffStorageIterator) Next() bool { + // If the iterator was already stale, consider it a programmer error. Although + // we could just return false here, triggering this path would probably mean + // somebody forgot to check for Error, so lets blow up instead of undefined + // behavior that's hard to debug. + if it.fail != nil { + panic(fmt.Sprintf("called Next of failed iterator: %v", it.fail)) + } + // Stop iterating if all keys were exhausted + if len(it.keys) == 0 { + return false + } + if it.layer.Stale() { + it.fail, it.keys = ErrSnapshotStale, nil + return false + } + // Iterator seems to be still alive, retrieve and cache the live hash + it.curHash = it.keys[0] + // key cached, shift the iterator and notify the user of success + it.keys = it.keys[1:] + return true +} + +// Error returns any failure that occurred during iteration, which might have +// caused a premature iteration exit (e.g. snapshot stack becoming stale). +func (it *diffStorageIterator) Error() error { + return it.fail +} + +// Hash returns the hash of the storage slot the iterator is currently at. +func (it *diffStorageIterator) Hash() common.Hash { + return it.curHash +} + +// Slot returns the raw storage slot value the iterator is currently at. +// This method may _fail_, if the underlying layer has been flattened between +// the call to Next and Value. That type of error will set it.Err. +// This method assumes that flattening does not delete elements from +// the storage mapping (writing nil into it is fine though), and will panic +// if elements have been deleted. +// +// Note the returned slot is not a copy, please don't modify it. +func (it *diffStorageIterator) Slot() []byte { + it.layer.lock.RLock() + storage, ok := it.layer.storageData[it.account] + if !ok { + panic(fmt.Sprintf("iterator referenced non-existent account storage: %x", it.account)) + } + // Storage slot might be nil(deleted), but it must exist + blob, ok := storage[it.curHash] + if !ok { + panic(fmt.Sprintf("iterator referenced non-existent storage slot: %x", it.curHash)) + } + it.layer.lock.RUnlock() + if it.layer.Stale() { + it.fail, it.keys = ErrSnapshotStale, nil + } + return blob +} + +// Release is a noop for diff account iterators as there are no held resources. +func (it *diffStorageIterator) Release() {} + +// diskStorageIterator is a storage iterator that steps over the live storage +// contained within a disk layer. +type diskStorageIterator struct { + layer *diskLayer + account common.Hash + it ethdb.Iterator +} + +// StorageIterator creates a storage iterator over a disk layer. +// If the whole storage is destructed, then all entries in the disk +// layer are deleted already. So the "destructed" flag returned here +// is always false. +func (dl *diskLayer) StorageIterator(account common.Hash, seek common.Hash) (StorageIterator, bool) { + pos := common.TrimRightZeroes(seek[:]) + return &diskStorageIterator{ + layer: dl, + account: account, + it: dl.diskdb.NewIterator(append(rawdb.SnapshotStoragePrefix, account.Bytes()...), pos), + }, false +} + +// Next steps the iterator forward one element, returning false if exhausted. +func (it *diskStorageIterator) Next() bool { + // If the iterator was already exhausted, don't bother + if it.it == nil { + return false + } + // Try to advance the iterator and release it if we reached the end + for { + if !it.it.Next() { + it.it.Release() + it.it = nil + return false + } + if len(it.it.Key()) == len(rawdb.SnapshotStoragePrefix)+common.HashLength+common.HashLength { + break + } + } + return true +} + +// Error returns any failure that occurred during iteration, which might have +// caused a premature iteration exit (e.g. snapshot stack becoming stale). +// +// A diff layer is immutable after creation content wise and can always be fully +// iterated without error, so this method always returns nil. +func (it *diskStorageIterator) Error() error { + if it.it == nil { + return nil // Iterator is exhausted and released + } + return it.it.Error() +} + +// Hash returns the hash of the storage slot the iterator is currently at. +func (it *diskStorageIterator) Hash() common.Hash { + return common.BytesToHash(it.it.Key()) // The prefix will be truncated +} + +// Slot returns the raw storage slot content the iterator is currently at. +func (it *diskStorageIterator) Slot() []byte { + return it.it.Value() +} + +// Release releases the database snapshot held during iteration. +func (it *diskStorageIterator) Release() { + // The iterator is auto-released on exhaustion, so make sure it's still alive + if it.it != nil { + it.it.Release() + it.it = nil + } +} diff --git a/core/state/snapshot/iterator_binary.go b/core/state/snapshot/iterator_binary.go new file mode 100644 index 0000000000..22184b2545 --- /dev/null +++ b/core/state/snapshot/iterator_binary.go @@ -0,0 +1,213 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "bytes" + + "github.com/ethereum/go-ethereum/common" +) + +// binaryIterator is a simplistic iterator to step over the accounts or storage +// in a snapshot, which may or may not be composed of multiple layers. Performance +// wise this iterator is slow, it's meant for cross validating the fast one, +type binaryIterator struct { + a Iterator + b Iterator + aDone bool + bDone bool + accountIterator bool + k common.Hash + account common.Hash + fail error +} + +// initBinaryAccountIterator creates a simplistic iterator to step over all the +// accounts in a slow, but easily verifiable way. Note this function is used for +// initialization, use `newBinaryAccountIterator` as the API. +func (dl *diffLayer) initBinaryAccountIterator() Iterator { + parent, ok := dl.parent.(*diffLayer) + if !ok { + l := &binaryIterator{ + a: dl.AccountIterator(common.Hash{}), + b: dl.Parent().AccountIterator(common.Hash{}), + accountIterator: true, + } + l.aDone = !l.a.Next() + l.bDone = !l.b.Next() + return l + } + l := &binaryIterator{ + a: dl.AccountIterator(common.Hash{}), + b: parent.initBinaryAccountIterator(), + accountIterator: true, + } + l.aDone = !l.a.Next() + l.bDone = !l.b.Next() + return l +} + +// initBinaryStorageIterator creates a simplistic iterator to step over all the +// storage slots in a slow, but easily verifiable way. Note this function is used +// for initialization, use `newBinaryStorageIterator` as the API. +func (dl *diffLayer) initBinaryStorageIterator(account common.Hash) Iterator { + parent, ok := dl.parent.(*diffLayer) + if !ok { + // If the storage in this layer is already destructed, discard all + // deeper layers but still return an valid single-branch iterator. + a, destructed := dl.StorageIterator(account, common.Hash{}) + if destructed { + l := &binaryIterator{ + a: a, + account: account, + } + l.aDone = !l.a.Next() + l.bDone = true + return l + } + // The parent is disk layer, don't need to take care "destructed" + // anymore. + b, _ := dl.Parent().StorageIterator(account, common.Hash{}) + l := &binaryIterator{ + a: a, + b: b, + account: account, + } + l.aDone = !l.a.Next() + l.bDone = !l.b.Next() + return l + } + // If the storage in this layer is already destructed, discard all + // deeper layers but still return an valid single-branch iterator. + a, destructed := dl.StorageIterator(account, common.Hash{}) + if destructed { + l := &binaryIterator{ + a: a, + account: account, + } + l.aDone = !l.a.Next() + l.bDone = true + return l + } + l := &binaryIterator{ + a: a, + b: parent.initBinaryStorageIterator(account), + account: account, + } + l.aDone = !l.a.Next() + l.bDone = !l.b.Next() + return l +} + +// Next steps the iterator forward one element, returning false if exhausted, +// or an error if iteration failed for some reason (e.g. root being iterated +// becomes stale and garbage collected). +func (it *binaryIterator) Next() bool { + if it.aDone && it.bDone { + return false + } +first: + if it.aDone { + it.k = it.b.Hash() + it.bDone = !it.b.Next() + return true + } + if it.bDone { + it.k = it.a.Hash() + it.aDone = !it.a.Next() + return true + } + nextA, nextB := it.a.Hash(), it.b.Hash() + if diff := bytes.Compare(nextA[:], nextB[:]); diff < 0 { + it.aDone = !it.a.Next() + it.k = nextA + return true + } else if diff == 0 { + // Now we need to advance one of them + it.aDone = !it.a.Next() + goto first + } + it.bDone = !it.b.Next() + it.k = nextB + return true +} + +// Error returns any failure that occurred during iteration, which might have +// caused a premature iteration exit (e.g. snapshot stack becoming stale). +func (it *binaryIterator) Error() error { + return it.fail +} + +// Hash returns the hash of the account the iterator is currently at. +func (it *binaryIterator) Hash() common.Hash { + return it.k +} + +// Account returns the RLP encoded slim account the iterator is currently at, or +// nil if the iterated snapshot stack became stale (you can check Error after +// to see if it failed or not). +// +// Note the returned account is not a copy, please don't modify it. +func (it *binaryIterator) Account() []byte { + if !it.accountIterator { + return nil + } + // The topmost iterator must be `diffAccountIterator` + blob, err := it.a.(*diffAccountIterator).layer.AccountRLP(it.k) + if err != nil { + it.fail = err + return nil + } + return blob +} + +// Slot returns the raw storage slot data the iterator is currently at, or +// nil if the iterated snapshot stack became stale (you can check Error after +// to see if it failed or not). +// +// Note the returned slot is not a copy, please don't modify it. +func (it *binaryIterator) Slot() []byte { + if it.accountIterator { + return nil + } + blob, err := it.a.(*diffStorageIterator).layer.Storage(it.account, it.k) + if err != nil { + it.fail = err + return nil + } + return blob +} + +// Release recursively releases all the iterators in the stack. +func (it *binaryIterator) Release() { + it.a.Release() + it.b.Release() +} + +// newBinaryAccountIterator creates a simplistic account iterator to step over +// all the accounts in a slow, but easily verifiable way. +func (dl *diffLayer) newBinaryAccountIterator() AccountIterator { + iter := dl.initBinaryAccountIterator() + return iter.(AccountIterator) +} + +// newBinaryStorageIterator creates a simplistic account iterator to step over +// all the storage slots in a slow, but easily verifiable way. +func (dl *diffLayer) newBinaryStorageIterator(account common.Hash) StorageIterator { + iter := dl.initBinaryStorageIterator(account) + return iter.(StorageIterator) +} diff --git a/core/state/snapshot/iterator_fast.go b/core/state/snapshot/iterator_fast.go new file mode 100644 index 0000000000..1a042c7cd3 --- /dev/null +++ b/core/state/snapshot/iterator_fast.go @@ -0,0 +1,350 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "bytes" + "fmt" + "sort" + + "github.com/ethereum/go-ethereum/common" +) + +// weightedIterator is a iterator with an assigned weight. It is used to prioritise +// which account or storage slot is the correct one if multiple iterators find the +// same one (modified in multiple consecutive blocks). +type weightedIterator struct { + it Iterator + priority int +} + +// weightedIterators is a set of iterators implementing the sort.Interface. +type weightedIterators []*weightedIterator + +// Len implements sort.Interface, returning the number of active iterators. +func (its weightedIterators) Len() int { return len(its) } + +// Less implements sort.Interface, returning which of two iterators in the stack +// is before the other. +func (its weightedIterators) Less(i, j int) bool { + // Order the iterators primarily by the account hashes + hashI := its[i].it.Hash() + hashJ := its[j].it.Hash() + + switch bytes.Compare(hashI[:], hashJ[:]) { + case -1: + return true + case 1: + return false + } + // Same account/storage-slot in multiple layers, split by priority + return its[i].priority < its[j].priority +} + +// Swap implements sort.Interface, swapping two entries in the iterator stack. +func (its weightedIterators) Swap(i, j int) { + its[i], its[j] = its[j], its[i] +} + +// fastIterator is a more optimized multi-layer iterator which maintains a +// direct mapping of all iterators leading down to the bottom layer. +type fastIterator struct { + tree *Tree // Snapshot tree to reinitialize stale sub-iterators with + root common.Hash // Root hash to reinitialize stale sub-iterators through + + curAccount []byte + curSlot []byte + + iterators weightedIterators + initiated bool + account bool + fail error +} + +// newFastIterator creates a new hierarchical account or storage iterator with one +// element per diff layer. The returned combo iterator can be used to walk over +// the entire snapshot diff stack simultaneously. +func newFastIterator(tree *Tree, root common.Hash, account common.Hash, seek common.Hash, accountIterator bool) (*fastIterator, error) { + snap := tree.Snapshot(root) + if snap == nil { + return nil, fmt.Errorf("unknown snapshot: %x", root) + } + fi := &fastIterator{ + tree: tree, + root: root, + account: accountIterator, + } + current := snap.(snapshot) + for depth := 0; current != nil; depth++ { + if accountIterator { + fi.iterators = append(fi.iterators, &weightedIterator{ + it: current.AccountIterator(seek), + priority: depth, + }) + } else { + // If the whole storage is destructed in this layer, don't + // bother deeper layer anymore. But we should still keep + // the iterator for this layer, since the iterator can contain + // some valid slots which belongs to the re-created account. + it, destructed := current.StorageIterator(account, seek) + fi.iterators = append(fi.iterators, &weightedIterator{ + it: it, + priority: depth, + }) + if destructed { + break + } + } + current = current.Parent() + } + fi.init() + return fi, nil +} + +// init walks over all the iterators and resolves any clashes between them, after +// which it prepares the stack for step-by-step iteration. +func (fi *fastIterator) init() { + // Track which account hashes are iterators positioned on + var positioned = make(map[common.Hash]int) + + // Position all iterators and track how many remain live + for i := 0; i < len(fi.iterators); i++ { + // Retrieve the first element and if it clashes with a previous iterator, + // advance either the current one or the old one. Repeat until nothing is + // clashing any more. + it := fi.iterators[i] + for { + // If the iterator is exhausted, drop it off the end + if !it.it.Next() { + it.it.Release() + last := len(fi.iterators) - 1 + + fi.iterators[i] = fi.iterators[last] + fi.iterators[last] = nil + fi.iterators = fi.iterators[:last] + + i-- + break + } + // The iterator is still alive, check for collisions with previous ones + hash := it.it.Hash() + if other, exist := positioned[hash]; !exist { + positioned[hash] = i + break + } else { + // Iterators collide, one needs to be progressed, use priority to + // determine which. + // + // This whole else-block can be avoided, if we instead + // do an initial priority-sort of the iterators. If we do that, + // then we'll only wind up here if a lower-priority (preferred) iterator + // has the same value, and then we will always just continue. + // However, it costs an extra sort, so it's probably not better + if fi.iterators[other].priority < it.priority { + // The 'it' should be progressed + continue + } else { + // The 'other' should be progressed, swap them + it = fi.iterators[other] + fi.iterators[other], fi.iterators[i] = fi.iterators[i], fi.iterators[other] + continue + } + } + } + } + // Re-sort the entire list + sort.Sort(fi.iterators) + fi.initiated = false +} + +// Next steps the iterator forward one element, returning false if exhausted. +func (fi *fastIterator) Next() bool { + if len(fi.iterators) == 0 { + return false + } + if !fi.initiated { + // Don't forward first time -- we had to 'Next' once in order to + // do the sorting already + fi.initiated = true + if fi.account { + fi.curAccount = fi.iterators[0].it.(AccountIterator).Account() + } else { + fi.curSlot = fi.iterators[0].it.(StorageIterator).Slot() + } + if innerErr := fi.iterators[0].it.Error(); innerErr != nil { + fi.fail = innerErr + return false + } + if fi.curAccount != nil || fi.curSlot != nil { + return true + } + // Implicit else: we've hit a nil-account or nil-slot, and need to + // fall through to the loop below to land on something non-nil + } + // If an account or a slot is deleted in one of the layers, the key will + // still be there, but the actual value will be nil. However, the iterator + // should not export nil-values (but instead simply omit the key), so we + // need to loop here until we either + // - get a non-nil value, + // - hit an error, + // - or exhaust the iterator + for { + if !fi.next(0) { + return false // exhausted + } + if fi.account { + fi.curAccount = fi.iterators[0].it.(AccountIterator).Account() + } else { + fi.curSlot = fi.iterators[0].it.(StorageIterator).Slot() + } + if innerErr := fi.iterators[0].it.Error(); innerErr != nil { + fi.fail = innerErr + return false // error + } + if fi.curAccount != nil || fi.curSlot != nil { + break // non-nil value found + } + } + return true +} + +// next handles the next operation internally and should be invoked when we know +// that two elements in the list may have the same value. +// +// For example, if the iterated hashes become [2,3,5,5,8,9,10], then we should +// invoke next(3), which will call Next on elem 3 (the second '5') and will +// cascade along the list, applying the same operation if needed. +func (fi *fastIterator) next(idx int) bool { + // If this particular iterator got exhausted, remove it and return true (the + // next one is surely not exhausted yet, otherwise it would have been removed + // already). + if it := fi.iterators[idx].it; !it.Next() { + it.Release() + + fi.iterators = append(fi.iterators[:idx], fi.iterators[idx+1:]...) + return len(fi.iterators) > 0 + } + // If there's no one left to cascade into, return + if idx == len(fi.iterators)-1 { + return true + } + // We next-ed the iterator at 'idx', now we may have to re-sort that element + var ( + cur, next = fi.iterators[idx], fi.iterators[idx+1] + curHash, nextHash = cur.it.Hash(), next.it.Hash() + ) + if diff := bytes.Compare(curHash[:], nextHash[:]); diff < 0 { + // It is still in correct place + return true + } else if diff == 0 && cur.priority < next.priority { + // So still in correct place, but we need to iterate on the next + fi.next(idx + 1) + return true + } + // At this point, the iterator is in the wrong location, but the remaining + // list is sorted. Find out where to move the item. + clash := -1 + index := sort.Search(len(fi.iterators), func(n int) bool { + // The iterator always advances forward, so anything before the old slot + // is known to be behind us, so just skip them altogether. This actually + // is an important clause since the sort order got invalidated. + if n < idx { + return false + } + if n == len(fi.iterators)-1 { + // Can always place an elem last + return true + } + nextHash := fi.iterators[n+1].it.Hash() + if diff := bytes.Compare(curHash[:], nextHash[:]); diff < 0 { + return true + } else if diff > 0 { + return false + } + // The elem we're placing it next to has the same value, + // so whichever winds up on n+1 will need further iteration + clash = n + 1 + + return cur.priority < fi.iterators[n+1].priority + }) + fi.move(idx, index) + if clash != -1 { + fi.next(clash) + } + return true +} + +// move advances an iterator to another position in the list. +func (fi *fastIterator) move(index, newpos int) { + elem := fi.iterators[index] + copy(fi.iterators[index:], fi.iterators[index+1:newpos+1]) + fi.iterators[newpos] = elem +} + +// Error returns any failure that occurred during iteration, which might have +// caused a premature iteration exit (e.g. snapshot stack becoming stale). +func (fi *fastIterator) Error() error { + return fi.fail +} + +// Hash returns the current key +func (fi *fastIterator) Hash() common.Hash { + return fi.iterators[0].it.Hash() +} + +// Account returns the current account blob. +// Note the returned account is not a copy, please don't modify it. +func (fi *fastIterator) Account() []byte { + return fi.curAccount +} + +// Slot returns the current storage slot. +// Note the returned slot is not a copy, please don't modify it. +func (fi *fastIterator) Slot() []byte { + return fi.curSlot +} + +// Release iterates over all the remaining live layer iterators and releases each +// of them individually. +func (fi *fastIterator) Release() { + for _, it := range fi.iterators { + it.it.Release() + } + fi.iterators = nil +} + +// Debug is a convenience helper during testing +func (fi *fastIterator) Debug() { + for _, it := range fi.iterators { + fmt.Printf("[p=%v v=%v] ", it.priority, it.it.Hash()[0]) + } + fmt.Println() +} + +// newFastAccountIterator creates a new hierarchical account iterator with one +// element per diff layer. The returned combo iterator can be used to walk over +// the entire snapshot diff stack simultaneously. +func newFastAccountIterator(tree *Tree, root common.Hash, seek common.Hash) (AccountIterator, error) { + return newFastIterator(tree, root, common.Hash{}, seek, true) +} + +// newFastStorageIterator creates a new hierarchical storage iterator with one +// element per diff layer. The returned combo iterator can be used to walk over +// the entire snapshot diff stack simultaneously. +func newFastStorageIterator(tree *Tree, root common.Hash, account common.Hash, seek common.Hash) (StorageIterator, error) { + return newFastIterator(tree, root, account, seek, false) +} diff --git a/core/state/snapshot/iterator_test.go b/core/state/snapshot/iterator_test.go new file mode 100644 index 0000000000..0296802d76 --- /dev/null +++ b/core/state/snapshot/iterator_test.go @@ -0,0 +1,1047 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "bytes" + crand "crypto/rand" + "encoding/binary" + "fmt" + "math/rand" + "testing" + + "github.com/VictoriaMetrics/fastcache" + "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/harmony/core/rawdb" +) + +// TestAccountIteratorBasics tests some simple single-layer(diff and disk) iteration +func TestAccountIteratorBasics(t *testing.T) { + var ( + destructs = make(map[common.Hash]struct{}) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) + ) + // Fill up a parent + for i := 0; i < 100; i++ { + h := randomHash() + data := randomAccount() + + accounts[h] = data + if rand.Intn(4) == 0 { + destructs[h] = struct{}{} + } + if rand.Intn(2) == 0 { + accStorage := make(map[common.Hash][]byte) + value := make([]byte, 32) + crand.Read(value) + accStorage[randomHash()] = value + storage[h] = accStorage + } + } + // Add some (identical) layers on top + diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) + it := diffLayer.AccountIterator(common.Hash{}) + verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator + + diskLayer := diffToDisk(diffLayer) + it = diskLayer.AccountIterator(common.Hash{}) + verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator +} + +// TestStorageIteratorBasics tests some simple single-layer(diff and disk) iteration for storage +func TestStorageIteratorBasics(t *testing.T) { + var ( + nilStorage = make(map[common.Hash]int) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) + ) + // Fill some random data + for i := 0; i < 10; i++ { + h := randomHash() + accounts[h] = randomAccount() + + accStorage := make(map[common.Hash][]byte) + value := make([]byte, 32) + + var nilstorage int + for i := 0; i < 100; i++ { + crand.Read(value) + if rand.Intn(2) == 0 { + accStorage[randomHash()] = common.CopyBytes(value) + } else { + accStorage[randomHash()] = nil // delete slot + nilstorage += 1 + } + } + storage[h] = accStorage + nilStorage[h] = nilstorage + } + // Add some (identical) layers on top + diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, nil, copyAccounts(accounts), copyStorage(storage)) + for account := range accounts { + it, _ := diffLayer.StorageIterator(account, common.Hash{}) + verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator + } + + diskLayer := diffToDisk(diffLayer) + for account := range accounts { + it, _ := diskLayer.StorageIterator(account, common.Hash{}) + verifyIterator(t, 100-nilStorage[account], it, verifyNothing) // Nil is allowed for single layer iterator + } +} + +type testIterator struct { + values []byte +} + +func newTestIterator(values ...byte) *testIterator { + return &testIterator{values} +} + +func (ti *testIterator) Seek(common.Hash) { + panic("implement me") +} + +func (ti *testIterator) Next() bool { + ti.values = ti.values[1:] + return len(ti.values) > 0 +} + +func (ti *testIterator) Error() error { + return nil +} + +func (ti *testIterator) Hash() common.Hash { + return common.BytesToHash([]byte{ti.values[0]}) +} + +func (ti *testIterator) Account() []byte { + return nil +} + +func (ti *testIterator) Slot() []byte { + return nil +} + +func (ti *testIterator) Release() {} + +func TestFastIteratorBasics(t *testing.T) { + type testCase struct { + lists [][]byte + expKeys []byte + } + for i, tc := range []testCase{ + {lists: [][]byte{{0, 1, 8}, {1, 2, 8}, {2, 9}, {4}, + {7, 14, 15}, {9, 13, 15, 16}}, + expKeys: []byte{0, 1, 2, 4, 7, 8, 9, 13, 14, 15, 16}}, + {lists: [][]byte{{0, 8}, {1, 2, 8}, {7, 14, 15}, {8, 9}, + {9, 10}, {10, 13, 15, 16}}, + expKeys: []byte{0, 1, 2, 7, 8, 9, 10, 13, 14, 15, 16}}, + } { + var iterators []*weightedIterator + for i, data := range tc.lists { + it := newTestIterator(data...) + iterators = append(iterators, &weightedIterator{it, i}) + } + fi := &fastIterator{ + iterators: iterators, + initiated: false, + } + count := 0 + for fi.Next() { + if got, exp := fi.Hash()[31], tc.expKeys[count]; exp != got { + t.Errorf("tc %d, [%d]: got %d exp %d", i, count, got, exp) + } + count++ + } + } +} + +type verifyContent int + +const ( + verifyNothing verifyContent = iota + verifyAccount + verifyStorage +) + +func verifyIterator(t *testing.T, expCount int, it Iterator, verify verifyContent) { + t.Helper() + + var ( + count = 0 + last = common.Hash{} + ) + for it.Next() { + hash := it.Hash() + if bytes.Compare(last[:], hash[:]) >= 0 { + t.Errorf("wrong order: %x >= %x", last, hash) + } + count++ + if verify == verifyAccount && len(it.(AccountIterator).Account()) == 0 { + t.Errorf("iterator returned nil-value for hash %x", hash) + } else if verify == verifyStorage && len(it.(StorageIterator).Slot()) == 0 { + t.Errorf("iterator returned nil-value for hash %x", hash) + } + last = hash + } + if count != expCount { + t.Errorf("iterator count mismatch: have %d, want %d", count, expCount) + } + if err := it.Error(); err != nil { + t.Errorf("iterator failed: %v", err) + } +} + +// TestAccountIteratorTraversal tests some simple multi-layer iteration. +func TestAccountIteratorTraversal(t *testing.T) { + // Create an empty base layer and a snapshot tree out of it + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + // Stack three diff layers on top with various overlaps + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil) + + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + randomAccountSet("0xbb", "0xdd", "0xf0"), nil) + + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + randomAccountSet("0xcc", "0xf0", "0xff"), nil) + + // Verify the single and multi-layer iterators + head := snaps.Snapshot(common.HexToHash("0x04")) + + verifyIterator(t, 3, head.(snapshot).AccountIterator(common.Hash{}), verifyNothing) + verifyIterator(t, 7, head.(*diffLayer).newBinaryAccountIterator(), verifyAccount) + + it, _ := snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{}) + verifyIterator(t, 7, it, verifyAccount) + it.Release() + + // Test after persist some bottom-most layers into the disk, + // the functionalities still work. + limit := aggregatorMemoryLimit + defer func() { + aggregatorMemoryLimit = limit + }() + aggregatorMemoryLimit = 0 // Force pushing the bottom-most layer into disk + snaps.Cap(common.HexToHash("0x04"), 2) + verifyIterator(t, 7, head.(*diffLayer).newBinaryAccountIterator(), verifyAccount) + + it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{}) + verifyIterator(t, 7, it, verifyAccount) + it.Release() +} + +func TestStorageIteratorTraversal(t *testing.T) { + // Create an empty base layer and a snapshot tree out of it + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + // Stack three diff layers on top with various overlaps + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x02", "0x03"}}, nil)) + + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x04", "0x05", "0x06"}}, nil)) + + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x02", "0x03"}}, nil)) + + // Verify the single and multi-layer iterators + head := snaps.Snapshot(common.HexToHash("0x04")) + + diffIter, _ := head.(snapshot).StorageIterator(common.HexToHash("0xaa"), common.Hash{}) + verifyIterator(t, 3, diffIter, verifyNothing) + verifyIterator(t, 6, head.(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa")), verifyStorage) + + it, _ := snaps.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.Hash{}) + verifyIterator(t, 6, it, verifyStorage) + it.Release() + + // Test after persist some bottom-most layers into the disk, + // the functionalities still work. + limit := aggregatorMemoryLimit + defer func() { + aggregatorMemoryLimit = limit + }() + aggregatorMemoryLimit = 0 // Force pushing the bottom-most layer into disk + snaps.Cap(common.HexToHash("0x04"), 2) + verifyIterator(t, 6, head.(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa")), verifyStorage) + + it, _ = snaps.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.Hash{}) + verifyIterator(t, 6, it, verifyStorage) + it.Release() +} + +// TestAccountIteratorTraversalValues tests some multi-layer iteration, where we +// also expect the correct values to show up. +func TestAccountIteratorTraversalValues(t *testing.T) { + // Create an empty base layer and a snapshot tree out of it + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + // Create a batch of account sets to seed subsequent layers with + var ( + a = make(map[common.Hash][]byte) + b = make(map[common.Hash][]byte) + c = make(map[common.Hash][]byte) + d = make(map[common.Hash][]byte) + e = make(map[common.Hash][]byte) + f = make(map[common.Hash][]byte) + g = make(map[common.Hash][]byte) + h = make(map[common.Hash][]byte) + ) + for i := byte(2); i < 0xff; i++ { + a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i)) + if i > 20 && i%2 == 0 { + b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i)) + } + if i%4 == 0 { + c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i)) + } + if i%7 == 0 { + d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i)) + } + if i%8 == 0 { + e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i)) + } + if i > 50 || i < 85 { + f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i)) + } + if i%64 == 0 { + g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i)) + } + if i%128 == 0 { + h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i)) + } + } + // Assemble a stack of snapshots from the account layers + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, a, nil) + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, b, nil) + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, c, nil) + snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, d, nil) + snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), nil, e, nil) + snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), nil, f, nil) + snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), nil, g, nil) + snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), nil, h, nil) + + it, _ := snaps.AccountIterator(common.HexToHash("0x09"), common.Hash{}) + head := snaps.Snapshot(common.HexToHash("0x09")) + for it.Next() { + hash := it.Hash() + want, err := head.AccountRLP(hash) + if err != nil { + t.Fatalf("failed to retrieve expected account: %v", err) + } + if have := it.Account(); !bytes.Equal(want, have) { + t.Fatalf("hash %x: account mismatch: have %x, want %x", hash, have, want) + } + } + it.Release() + + // Test after persist some bottom-most layers into the disk, + // the functionalities still work. + limit := aggregatorMemoryLimit + defer func() { + aggregatorMemoryLimit = limit + }() + aggregatorMemoryLimit = 0 // Force pushing the bottom-most layer into disk + snaps.Cap(common.HexToHash("0x09"), 2) + + it, _ = snaps.AccountIterator(common.HexToHash("0x09"), common.Hash{}) + for it.Next() { + hash := it.Hash() + want, err := head.AccountRLP(hash) + if err != nil { + t.Fatalf("failed to retrieve expected account: %v", err) + } + if have := it.Account(); !bytes.Equal(want, have) { + t.Fatalf("hash %x: account mismatch: have %x, want %x", hash, have, want) + } + } + it.Release() +} + +func TestStorageIteratorTraversalValues(t *testing.T) { + // Create an empty base layer and a snapshot tree out of it + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + wrapStorage := func(storage map[common.Hash][]byte) map[common.Hash]map[common.Hash][]byte { + return map[common.Hash]map[common.Hash][]byte{ + common.HexToHash("0xaa"): storage, + } + } + // Create a batch of storage sets to seed subsequent layers with + var ( + a = make(map[common.Hash][]byte) + b = make(map[common.Hash][]byte) + c = make(map[common.Hash][]byte) + d = make(map[common.Hash][]byte) + e = make(map[common.Hash][]byte) + f = make(map[common.Hash][]byte) + g = make(map[common.Hash][]byte) + h = make(map[common.Hash][]byte) + ) + for i := byte(2); i < 0xff; i++ { + a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i)) + if i > 20 && i%2 == 0 { + b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i)) + } + if i%4 == 0 { + c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i)) + } + if i%7 == 0 { + d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i)) + } + if i%8 == 0 { + e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i)) + } + if i > 50 || i < 85 { + f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i)) + } + if i%64 == 0 { + g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i)) + } + if i%128 == 0 { + h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i)) + } + } + // Assemble a stack of snapshots from the account layers + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, randomAccountSet("0xaa"), wrapStorage(a)) + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, randomAccountSet("0xaa"), wrapStorage(b)) + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, randomAccountSet("0xaa"), wrapStorage(c)) + snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, randomAccountSet("0xaa"), wrapStorage(d)) + snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), nil, randomAccountSet("0xaa"), wrapStorage(e)) + snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), nil, randomAccountSet("0xaa"), wrapStorage(e)) + snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), nil, randomAccountSet("0xaa"), wrapStorage(g)) + snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), nil, randomAccountSet("0xaa"), wrapStorage(h)) + + it, _ := snaps.StorageIterator(common.HexToHash("0x09"), common.HexToHash("0xaa"), common.Hash{}) + head := snaps.Snapshot(common.HexToHash("0x09")) + for it.Next() { + hash := it.Hash() + want, err := head.Storage(common.HexToHash("0xaa"), hash) + if err != nil { + t.Fatalf("failed to retrieve expected storage slot: %v", err) + } + if have := it.Slot(); !bytes.Equal(want, have) { + t.Fatalf("hash %x: slot mismatch: have %x, want %x", hash, have, want) + } + } + it.Release() + + // Test after persist some bottom-most layers into the disk, + // the functionalities still work. + limit := aggregatorMemoryLimit + defer func() { + aggregatorMemoryLimit = limit + }() + aggregatorMemoryLimit = 0 // Force pushing the bottom-most layer into disk + snaps.Cap(common.HexToHash("0x09"), 2) + + it, _ = snaps.StorageIterator(common.HexToHash("0x09"), common.HexToHash("0xaa"), common.Hash{}) + for it.Next() { + hash := it.Hash() + want, err := head.Storage(common.HexToHash("0xaa"), hash) + if err != nil { + t.Fatalf("failed to retrieve expected slot: %v", err) + } + if have := it.Slot(); !bytes.Equal(want, have) { + t.Fatalf("hash %x: slot mismatch: have %x, want %x", hash, have, want) + } + } + it.Release() +} + +// This testcase is notorious, all layers contain the exact same 200 accounts. +func TestAccountIteratorLargeTraversal(t *testing.T) { + // Create a custom account factory to recreate the same addresses + makeAccounts := func(num int) map[common.Hash][]byte { + accounts := make(map[common.Hash][]byte) + for i := 0; i < num; i++ { + h := common.Hash{} + binary.BigEndian.PutUint64(h[:], uint64(i+1)) + accounts[h] = randomAccount() + } + return accounts + } + // Build up a large stack of snapshots + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + for i := 1; i < 128; i++ { + snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(200), nil) + } + // Iterate the entire stack and ensure everything is hit only once + head := snaps.Snapshot(common.HexToHash("0x80")) + verifyIterator(t, 200, head.(snapshot).AccountIterator(common.Hash{}), verifyNothing) + verifyIterator(t, 200, head.(*diffLayer).newBinaryAccountIterator(), verifyAccount) + + it, _ := snaps.AccountIterator(common.HexToHash("0x80"), common.Hash{}) + verifyIterator(t, 200, it, verifyAccount) + it.Release() + + // Test after persist some bottom-most layers into the disk, + // the functionalities still work. + limit := aggregatorMemoryLimit + defer func() { + aggregatorMemoryLimit = limit + }() + aggregatorMemoryLimit = 0 // Force pushing the bottom-most layer into disk + snaps.Cap(common.HexToHash("0x80"), 2) + + verifyIterator(t, 200, head.(*diffLayer).newBinaryAccountIterator(), verifyAccount) + + it, _ = snaps.AccountIterator(common.HexToHash("0x80"), common.Hash{}) + verifyIterator(t, 200, it, verifyAccount) + it.Release() +} + +// TestAccountIteratorFlattening tests what happens when we +// - have a live iterator on child C (parent C1 -> C2 .. CN) +// - flattens C2 all the way into CN +// - continues iterating +func TestAccountIteratorFlattening(t *testing.T) { + // Create an empty base layer and a snapshot tree out of it + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + // Create a stack of diffs on top + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil) + + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + randomAccountSet("0xbb", "0xdd", "0xf0"), nil) + + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + randomAccountSet("0xcc", "0xf0", "0xff"), nil) + + // Create an iterator and flatten the data from underneath it + it, _ := snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{}) + defer it.Release() + + if err := snaps.Cap(common.HexToHash("0x04"), 1); err != nil { + t.Fatalf("failed to flatten snapshot stack: %v", err) + } + //verifyIterator(t, 7, it) +} + +func TestAccountIteratorSeek(t *testing.T) { + // Create a snapshot stack with some initial data + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil) + + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + randomAccountSet("0xbb", "0xdd", "0xf0"), nil) + + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + randomAccountSet("0xcc", "0xf0", "0xff"), nil) + + // Account set is now + // 02: aa, ee, f0, ff + // 03: aa, bb, dd, ee, f0 (, f0), ff + // 04: aa, bb, cc, dd, ee, f0 (, f0), ff (, ff) + // Construct various iterators and ensure their traversal is correct + it, _ := snaps.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xdd")) + defer it.Release() + verifyIterator(t, 3, it, verifyAccount) // expected: ee, f0, ff + + it, _ = snaps.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xaa")) + defer it.Release() + verifyIterator(t, 4, it, verifyAccount) // expected: aa, ee, f0, ff + + it, _ = snaps.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xff")) + defer it.Release() + verifyIterator(t, 1, it, verifyAccount) // expected: ff + + it, _ = snaps.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xff1")) + defer it.Release() + verifyIterator(t, 0, it, verifyAccount) // expected: nothing + + it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xbb")) + defer it.Release() + verifyIterator(t, 6, it, verifyAccount) // expected: bb, cc, dd, ee, f0, ff + + it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xef")) + defer it.Release() + verifyIterator(t, 2, it, verifyAccount) // expected: f0, ff + + it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xf0")) + defer it.Release() + verifyIterator(t, 2, it, verifyAccount) // expected: f0, ff + + it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xff")) + defer it.Release() + verifyIterator(t, 1, it, verifyAccount) // expected: ff + + it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xff1")) + defer it.Release() + verifyIterator(t, 0, it, verifyAccount) // expected: nothing +} + +func TestStorageIteratorSeek(t *testing.T) { + // Create a snapshot stack with some initial data + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + // Stack three diff layers on top with various overlaps + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x03", "0x05"}}, nil)) + + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x02", "0x05", "0x06"}}, nil)) + + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x05", "0x08"}}, nil)) + + // Account set is now + // 02: 01, 03, 05 + // 03: 01, 02, 03, 05 (, 05), 06 + // 04: 01(, 01), 02, 03, 05(, 05, 05), 06, 08 + // Construct various iterators and ensure their traversal is correct + it, _ := snaps.StorageIterator(common.HexToHash("0x02"), common.HexToHash("0xaa"), common.HexToHash("0x01")) + defer it.Release() + verifyIterator(t, 3, it, verifyStorage) // expected: 01, 03, 05 + + it, _ = snaps.StorageIterator(common.HexToHash("0x02"), common.HexToHash("0xaa"), common.HexToHash("0x02")) + defer it.Release() + verifyIterator(t, 2, it, verifyStorage) // expected: 03, 05 + + it, _ = snaps.StorageIterator(common.HexToHash("0x02"), common.HexToHash("0xaa"), common.HexToHash("0x5")) + defer it.Release() + verifyIterator(t, 1, it, verifyStorage) // expected: 05 + + it, _ = snaps.StorageIterator(common.HexToHash("0x02"), common.HexToHash("0xaa"), common.HexToHash("0x6")) + defer it.Release() + verifyIterator(t, 0, it, verifyStorage) // expected: nothing + + it, _ = snaps.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.HexToHash("0x01")) + defer it.Release() + verifyIterator(t, 6, it, verifyStorage) // expected: 01, 02, 03, 05, 06, 08 + + it, _ = snaps.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.HexToHash("0x05")) + defer it.Release() + verifyIterator(t, 3, it, verifyStorage) // expected: 05, 06, 08 + + it, _ = snaps.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.HexToHash("0x08")) + defer it.Release() + verifyIterator(t, 1, it, verifyStorage) // expected: 08 + + it, _ = snaps.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.HexToHash("0x09")) + defer it.Release() + verifyIterator(t, 0, it, verifyStorage) // expected: nothing +} + +// TestAccountIteratorDeletions tests that the iterator behaves correct when there are +// deleted accounts (where the Account() value is nil). The iterator +// should not output any accounts or nil-values for those cases. +func TestAccountIteratorDeletions(t *testing.T) { + // Create an empty base layer and a snapshot tree out of it + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + // Stack three diff layers on top with various overlaps + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), + nil, randomAccountSet("0x11", "0x22", "0x33"), nil) + + deleted := common.HexToHash("0x22") + destructed := map[common.Hash]struct{}{ + deleted: {}, + } + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), + destructed, randomAccountSet("0x11", "0x33"), nil) + + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), + nil, randomAccountSet("0x33", "0x44", "0x55"), nil) + + // The output should be 11,33,44,55 + it, _ := snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{}) + // Do a quick check + verifyIterator(t, 4, it, verifyAccount) + it.Release() + + // And a more detailed verification that we indeed do not see '0x22' + it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{}) + defer it.Release() + for it.Next() { + hash := it.Hash() + if it.Account() == nil { + t.Errorf("iterator returned nil-value for hash %x", hash) + } + if hash == deleted { + t.Errorf("expected deleted elem %x to not be returned by iterator", deleted) + } + } +} + +func TestStorageIteratorDeletions(t *testing.T) { + // Create an empty base layer and a snapshot tree out of it + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + // Stack three diff layers on top with various overlaps + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x03", "0x05"}}, nil)) + + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x02", "0x04", "0x06"}}, [][]string{{"0x01", "0x03"}})) + + // The output should be 02,04,05,06 + it, _ := snaps.StorageIterator(common.HexToHash("0x03"), common.HexToHash("0xaa"), common.Hash{}) + verifyIterator(t, 4, it, verifyStorage) + it.Release() + + // The output should be 04,05,06 + it, _ = snaps.StorageIterator(common.HexToHash("0x03"), common.HexToHash("0xaa"), common.HexToHash("0x03")) + verifyIterator(t, 3, it, verifyStorage) + it.Release() + + // Destruct the whole storage + destructed := map[common.Hash]struct{}{ + common.HexToHash("0xaa"): {}, + } + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), destructed, nil, nil) + + it, _ = snaps.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.Hash{}) + verifyIterator(t, 0, it, verifyStorage) + it.Release() + + // Re-insert the slots of the same account + snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, + randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x07", "0x08", "0x09"}}, nil)) + + // The output should be 07,08,09 + it, _ = snaps.StorageIterator(common.HexToHash("0x05"), common.HexToHash("0xaa"), common.Hash{}) + verifyIterator(t, 3, it, verifyStorage) + it.Release() + + // Destruct the whole storage but re-create the account in the same layer + snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), destructed, randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x11", "0x12"}}, nil)) + it, _ = snaps.StorageIterator(common.HexToHash("0x06"), common.HexToHash("0xaa"), common.Hash{}) + verifyIterator(t, 2, it, verifyStorage) // The output should be 11,12 + it.Release() + + verifyIterator(t, 2, snaps.Snapshot(common.HexToHash("0x06")).(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa")), verifyStorage) +} + +// BenchmarkAccountIteratorTraversal is a bit a bit notorious -- all layers contain the +// exact same 200 accounts. That means that we need to process 2000 items, but +// only spit out 200 values eventually. +// +// The value-fetching benchmark is easy on the binary iterator, since it never has to reach +// down at any depth for retrieving the values -- all are on the topmost layer +// +// BenchmarkAccountIteratorTraversal/binary_iterator_keys-6 2239 483674 ns/op +// BenchmarkAccountIteratorTraversal/binary_iterator_values-6 2403 501810 ns/op +// BenchmarkAccountIteratorTraversal/fast_iterator_keys-6 1923 677966 ns/op +// BenchmarkAccountIteratorTraversal/fast_iterator_values-6 1741 649967 ns/op +func BenchmarkAccountIteratorTraversal(b *testing.B) { + // Create a custom account factory to recreate the same addresses + makeAccounts := func(num int) map[common.Hash][]byte { + accounts := make(map[common.Hash][]byte) + for i := 0; i < num; i++ { + h := common.Hash{} + binary.BigEndian.PutUint64(h[:], uint64(i+1)) + accounts[h] = randomAccount() + } + return accounts + } + // Build up a large stack of snapshots + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + for i := 1; i <= 100; i++ { + snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(200), nil) + } + // We call this once before the benchmark, so the creation of + // sorted accountlists are not included in the results. + head := snaps.Snapshot(common.HexToHash("0x65")) + head.(*diffLayer).newBinaryAccountIterator() + + b.Run("binary iterator keys", func(b *testing.B) { + for i := 0; i < b.N; i++ { + got := 0 + it := head.(*diffLayer).newBinaryAccountIterator() + for it.Next() { + got++ + } + if exp := 200; got != exp { + b.Errorf("iterator len wrong, expected %d, got %d", exp, got) + } + } + }) + b.Run("binary iterator values", func(b *testing.B) { + for i := 0; i < b.N; i++ { + got := 0 + it := head.(*diffLayer).newBinaryAccountIterator() + for it.Next() { + got++ + head.(*diffLayer).accountRLP(it.Hash(), 0) + } + if exp := 200; got != exp { + b.Errorf("iterator len wrong, expected %d, got %d", exp, got) + } + } + }) + b.Run("fast iterator keys", func(b *testing.B) { + for i := 0; i < b.N; i++ { + it, _ := snaps.AccountIterator(common.HexToHash("0x65"), common.Hash{}) + defer it.Release() + + got := 0 + for it.Next() { + got++ + } + if exp := 200; got != exp { + b.Errorf("iterator len wrong, expected %d, got %d", exp, got) + } + } + }) + b.Run("fast iterator values", func(b *testing.B) { + for i := 0; i < b.N; i++ { + it, _ := snaps.AccountIterator(common.HexToHash("0x65"), common.Hash{}) + defer it.Release() + + got := 0 + for it.Next() { + got++ + it.Account() + } + if exp := 200; got != exp { + b.Errorf("iterator len wrong, expected %d, got %d", exp, got) + } + } + }) +} + +// BenchmarkAccountIteratorLargeBaselayer is a pretty realistic benchmark, where +// the baselayer is a lot larger than the upper layer. +// +// This is heavy on the binary iterator, which in most cases will have to +// call recursively 100 times for the majority of the values +// +// BenchmarkAccountIteratorLargeBaselayer/binary_iterator_(keys)-6 514 1971999 ns/op +// BenchmarkAccountIteratorLargeBaselayer/binary_iterator_(values)-6 61 18997492 ns/op +// BenchmarkAccountIteratorLargeBaselayer/fast_iterator_(keys)-6 10000 114385 ns/op +// BenchmarkAccountIteratorLargeBaselayer/fast_iterator_(values)-6 4047 296823 ns/op +func BenchmarkAccountIteratorLargeBaselayer(b *testing.B) { + // Create a custom account factory to recreate the same addresses + makeAccounts := func(num int) map[common.Hash][]byte { + accounts := make(map[common.Hash][]byte) + for i := 0; i < num; i++ { + h := common.Hash{} + binary.BigEndian.PutUint64(h[:], uint64(i+1)) + accounts[h] = randomAccount() + } + return accounts + } + // Build up a large stack of snapshots + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, makeAccounts(2000), nil) + for i := 2; i <= 100; i++ { + snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(20), nil) + } + // We call this once before the benchmark, so the creation of + // sorted accountlists are not included in the results. + head := snaps.Snapshot(common.HexToHash("0x65")) + head.(*diffLayer).newBinaryAccountIterator() + + b.Run("binary iterator (keys)", func(b *testing.B) { + for i := 0; i < b.N; i++ { + got := 0 + it := head.(*diffLayer).newBinaryAccountIterator() + for it.Next() { + got++ + } + if exp := 2000; got != exp { + b.Errorf("iterator len wrong, expected %d, got %d", exp, got) + } + } + }) + b.Run("binary iterator (values)", func(b *testing.B) { + for i := 0; i < b.N; i++ { + got := 0 + it := head.(*diffLayer).newBinaryAccountIterator() + for it.Next() { + got++ + v := it.Hash() + head.(*diffLayer).accountRLP(v, 0) + } + if exp := 2000; got != exp { + b.Errorf("iterator len wrong, expected %d, got %d", exp, got) + } + } + }) + b.Run("fast iterator (keys)", func(b *testing.B) { + for i := 0; i < b.N; i++ { + it, _ := snaps.AccountIterator(common.HexToHash("0x65"), common.Hash{}) + defer it.Release() + + got := 0 + for it.Next() { + got++ + } + if exp := 2000; got != exp { + b.Errorf("iterator len wrong, expected %d, got %d", exp, got) + } + } + }) + b.Run("fast iterator (values)", func(b *testing.B) { + for i := 0; i < b.N; i++ { + it, _ := snaps.AccountIterator(common.HexToHash("0x65"), common.Hash{}) + defer it.Release() + + got := 0 + for it.Next() { + it.Account() + got++ + } + if exp := 2000; got != exp { + b.Errorf("iterator len wrong, expected %d, got %d", exp, got) + } + } + }) +} + +/* +func BenchmarkBinaryAccountIteration(b *testing.B) { + benchmarkAccountIteration(b, func(snap snapshot) AccountIterator { + return snap.(*diffLayer).newBinaryAccountIterator() + }) +} + +func BenchmarkFastAccountIteration(b *testing.B) { + benchmarkAccountIteration(b, newFastAccountIterator) +} + +func benchmarkAccountIteration(b *testing.B, iterator func(snap snapshot) AccountIterator) { + // Create a diff stack and randomize the accounts across them + layers := make([]map[common.Hash][]byte, 128) + for i := 0; i < len(layers); i++ { + layers[i] = make(map[common.Hash][]byte) + } + for i := 0; i < b.N; i++ { + depth := rand.Intn(len(layers)) + layers[depth][randomHash()] = randomAccount() + } + stack := snapshot(emptyLayer()) + for _, layer := range layers { + stack = stack.Update(common.Hash{}, layer, nil, nil) + } + // Reset the timers and report all the stats + it := iterator(stack) + + b.ResetTimer() + b.ReportAllocs() + + for it.Next() { + } +} +*/ diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go new file mode 100644 index 0000000000..a724581164 --- /dev/null +++ b/core/state/snapshot/journal.go @@ -0,0 +1,374 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "time" + + "github.com/VictoriaMetrics/fastcache" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/harmony-one/harmony/core/rawdb" + "github.com/harmony-one/harmony/internal/utils" +) + +const journalVersion uint64 = 0 + +// journalGenerator is a disk layer entry containing the generator progress marker. +type journalGenerator struct { + // Indicator that whether the database was in progress of being wiped. + // It's deprecated but keep it here for background compatibility. + Wiping bool + + Done bool // Whether the generator finished creating the snapshot + Marker []byte + Accounts uint64 + Slots uint64 + Storage uint64 +} + +// journalDestruct is an account deletion entry in a diffLayer's disk journal. +type journalDestruct struct { + Hash common.Hash +} + +// journalAccount is an account entry in a diffLayer's disk journal. +type journalAccount struct { + Hash common.Hash + Blob []byte +} + +// journalStorage is an account's storage map in a diffLayer's disk journal. +type journalStorage struct { + Hash common.Hash + Keys []common.Hash + Vals [][]byte +} + +func ParseGeneratorStatus(generatorBlob []byte) string { + if len(generatorBlob) == 0 { + return "" + } + var generator journalGenerator + if err := rlp.DecodeBytes(generatorBlob, &generator); err != nil { + utils.Logger().Warn().Err(err).Msg("failed to decode snapshot generator") + return "" + } + // Figure out whether we're after or within an account + var m string + switch marker := generator.Marker; len(marker) { + case common.HashLength: + m = fmt.Sprintf("at %#x", marker) + case 2 * common.HashLength: + m = fmt.Sprintf("in %#x at %#x", marker[:common.HashLength], marker[common.HashLength:]) + default: + m = fmt.Sprintf("%#x", marker) + } + return fmt.Sprintf(`Done: %v, Accounts: %d, Slots: %d, Storage: %d, Marker: %s`, + generator.Done, generator.Accounts, generator.Slots, generator.Storage, m) +} + +// loadAndParseJournal tries to parse the snapshot journal in latest format. +func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, journalGenerator, error) { + // Retrieve the disk layer generator. It must exist, no matter the + // snapshot is fully generated or not. Otherwise the entire disk + // layer is invalid. + generatorBlob := rawdb.ReadSnapshotGenerator(db) + if len(generatorBlob) == 0 { + return nil, journalGenerator{}, errors.New("missing snapshot generator") + } + var generator journalGenerator + if err := rlp.DecodeBytes(generatorBlob, &generator); err != nil { + return nil, journalGenerator{}, fmt.Errorf("failed to decode snapshot generator: %v", err) + } + // Retrieve the diff layer journal. It's possible that the journal is + // not existent, e.g. the disk layer is generating while that the Geth + // crashes without persisting the diff journal. + // So if there is no journal, or the journal is invalid(e.g. the journal + // is not matched with disk layer; or the it's the legacy-format journal, + // etc.), we just discard all diffs and try to recover them later. + var current snapshot = base + err := iterateJournal(db, func(parent common.Hash, root common.Hash, destructSet map[common.Hash]struct{}, accountData map[common.Hash][]byte, storageData map[common.Hash]map[common.Hash][]byte) error { + current = newDiffLayer(current, root, destructSet, accountData, storageData) + return nil + }) + if err != nil { + return base, generator, nil + } + return current, generator, nil +} + +// loadSnapshot loads a pre-existing state snapshot backed by a key-value store. +func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, root common.Hash, cache int, recovery bool, noBuild bool) (snapshot, bool, error) { + // If snapshotting is disabled (initial sync in progress), don't do anything, + // wait for the chain to permit us to do something meaningful + if rawdb.ReadSnapshotDisabled(diskdb) { + return nil, true, nil + } + // Retrieve the block number and hash of the snapshot, failing if no snapshot + // is present in the database (or crashed mid-update). + baseRoot := rawdb.ReadSnapshotRoot(diskdb) + if baseRoot == (common.Hash{}) { + return nil, false, errors.New("missing or corrupted snapshot") + } + base := &diskLayer{ + diskdb: diskdb, + triedb: triedb, + cache: fastcache.New(cache * 1024 * 1024), + root: baseRoot, + } + snapshot, generator, err := loadAndParseJournal(diskdb, base) + if err != nil { + utils.Logger().Warn().Err(err).Msg("Failed to load journal") + return nil, false, err + } + // Entire snapshot journal loaded, sanity check the head. If the loaded + // snapshot is not matched with current state root, print a warning log + // or discard the entire snapshot it's legacy snapshot. + // + // Possible scenario: Geth was crashed without persisting journal and then + // restart, the head is rewound to the point with available state(trie) + // which is below the snapshot. In this case the snapshot can be recovered + // by re-executing blocks but right now it's unavailable. + if head := snapshot.Root(); head != root { + // If it's legacy snapshot, or it's new-format snapshot but + // it's not in recovery mode, returns the error here for + // rebuilding the entire snapshot forcibly. + if !recovery { + return nil, false, fmt.Errorf("head doesn't match snapshot: have %#x, want %#x", head, root) + } + // It's in snapshot recovery, the assumption is held that + // the disk layer is always higher than chain head. It can + // be eventually recovered when the chain head beyonds the + // disk layer. + utils.Logger().Warn().Err(err). + Interface("snaproot", head). + Interface("chainroot", root). + Msg("Snapshot is not continuous with chain") + } + // Load the disk layer status from the generator if it's not complete + if !generator.Done { + base.genMarker = generator.Marker + if base.genMarker == nil { + base.genMarker = []byte{} + } + } + // Everything loaded correctly, resume any suspended operations + // if the background generation is allowed + if !generator.Done && !noBuild { + base.genPending = make(chan struct{}) + base.genAbort = make(chan chan *generatorStats) + + var origin uint64 + if len(generator.Marker) >= 8 { + origin = binary.BigEndian.Uint64(generator.Marker) + } + go base.generate(&generatorStats{ + origin: origin, + start: time.Now(), + accounts: generator.Accounts, + slots: generator.Slots, + storage: common.StorageSize(generator.Storage), + }) + } + return snapshot, false, nil +} + +// Journal terminates any in-progress snapshot generation, also implicitly pushing +// the progress into the database. +func (dl *diskLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { + // If the snapshot is currently being generated, abort it + var stats *generatorStats + if dl.genAbort != nil { + abort := make(chan *generatorStats) + dl.genAbort <- abort + + if stats = <-abort; stats != nil { + stats.Log("Journalling in-progress snapshot", dl.root, dl.genMarker) + } + } + // Ensure the layer didn't get stale + dl.lock.RLock() + defer dl.lock.RUnlock() + + if dl.stale { + return common.Hash{}, ErrSnapshotStale + } + // Ensure the generator stats is written even if none was ran this cycle + journalProgress(dl.diskdb, dl.genMarker, stats) + + utils.Logger().Debug().Interface("root", dl.root).Msg("Journalled disk layer") + return dl.root, nil +} + +// Journal writes the memory layer contents into a buffer to be stored in the +// database as the snapshot journal. +func (dl *diffLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { + // Journal the parent first + base, err := dl.parent.Journal(buffer) + if err != nil { + return common.Hash{}, err + } + // Ensure the layer didn't get stale + dl.lock.RLock() + defer dl.lock.RUnlock() + + if dl.Stale() { + return common.Hash{}, ErrSnapshotStale + } + // Everything below was journalled, persist this layer too + if err := rlp.Encode(buffer, dl.root); err != nil { + return common.Hash{}, err + } + destructs := make([]journalDestruct, 0, len(dl.destructSet)) + for hash := range dl.destructSet { + destructs = append(destructs, journalDestruct{Hash: hash}) + } + if err := rlp.Encode(buffer, destructs); err != nil { + return common.Hash{}, err + } + accounts := make([]journalAccount, 0, len(dl.accountData)) + for hash, blob := range dl.accountData { + accounts = append(accounts, journalAccount{Hash: hash, Blob: blob}) + } + if err := rlp.Encode(buffer, accounts); err != nil { + return common.Hash{}, err + } + storage := make([]journalStorage, 0, len(dl.storageData)) + for hash, slots := range dl.storageData { + keys := make([]common.Hash, 0, len(slots)) + vals := make([][]byte, 0, len(slots)) + for key, val := range slots { + keys = append(keys, key) + vals = append(vals, val) + } + storage = append(storage, journalStorage{Hash: hash, Keys: keys, Vals: vals}) + } + if err := rlp.Encode(buffer, storage); err != nil { + return common.Hash{}, err + } + utils.Logger().Debug().Err(err).Interface("root", dl.root).Interface("parent", dl.parent.Root()).Msg("Journalled diff layer") + return base, nil +} + +// journalCallback is a function which is invoked by iterateJournal, every +// time a difflayer is loaded from disk. +type journalCallback = func(parent common.Hash, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error + +// iterateJournal iterates through the journalled difflayers, loading them from +// the database, and invoking the callback for each loaded layer. +// The order is incremental; starting with the bottom-most difflayer, going towards +// the most recent layer. +// This method returns error either if there was some error reading from disk, +// OR if the callback returns an error when invoked. +func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { + journal := rawdb.ReadSnapshotJournal(db) + if len(journal) == 0 { + utils.Logger().Warn().Str("diffs", "missing").Msg("Loaded snapshot journal") + return nil + } + r := rlp.NewStream(bytes.NewReader(journal), 0) + // Firstly, resolve the first element as the journal version + version, err := r.Uint64() + if err != nil { + utils.Logger().Warn().Err(err).Msg("Failed to resolve the journal version") + return errors.New("failed to resolve journal version") + } + if version != journalVersion { + utils.Logger().Warn().Err(err). + Uint64("required", journalVersion). + Uint64("got", version). + Msg("Discarded the snapshot journal with wrong version") + + return errors.New("wrong journal version") + } + // Secondly, resolve the disk layer root, ensure it's continuous + // with disk layer. Note now we can ensure it's the snapshot journal + // correct version, so we expect everything can be resolved properly. + var parent common.Hash + if err := r.Decode(&parent); err != nil { + return errors.New("missing disk layer root") + } + if baseRoot := rawdb.ReadSnapshotRoot(db); baseRoot != parent { + utils.Logger().Warn().Err(err). + Interface("disk_root", baseRoot). + Str("diffs", "unmatched"). + Msg("Loaded snapshot journal") + + return fmt.Errorf("mismatched disk and diff layers") + } + for { + var ( + root common.Hash + destructs []journalDestruct + accounts []journalAccount + storage []journalStorage + destructSet = make(map[common.Hash]struct{}) + accountData = make(map[common.Hash][]byte) + storageData = make(map[common.Hash]map[common.Hash][]byte) + ) + // Read the next diff journal entry + if err := r.Decode(&root); err != nil { + // The first read may fail with EOF, marking the end of the journal + if errors.Is(err, io.EOF) { + return nil + } + return fmt.Errorf("load diff root: %v", err) + } + if err := r.Decode(&destructs); err != nil { + return fmt.Errorf("load diff destructs: %v", err) + } + if err := r.Decode(&accounts); err != nil { + return fmt.Errorf("load diff accounts: %v", err) + } + if err := r.Decode(&storage); err != nil { + return fmt.Errorf("load diff storage: %v", err) + } + for _, entry := range destructs { + destructSet[entry.Hash] = struct{}{} + } + for _, entry := range accounts { + if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that + accountData[entry.Hash] = entry.Blob + } else { + accountData[entry.Hash] = nil + } + } + for _, entry := range storage { + slots := make(map[common.Hash][]byte) + for i, key := range entry.Keys { + if len(entry.Vals[i]) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that + slots[key] = entry.Vals[i] + } else { + slots[key] = nil + } + } + storageData[entry.Hash] = slots + } + if err := callback(parent, root, destructSet, accountData, storageData); err != nil { + return err + } + parent = root + } +} diff --git a/core/state/snapshot/metrics.go b/core/state/snapshot/metrics.go new file mode 100644 index 0000000000..b2e884588b --- /dev/null +++ b/core/state/snapshot/metrics.go @@ -0,0 +1,53 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import "github.com/ethereum/go-ethereum/metrics" + +// Metrics in generation +var ( + snapGeneratedAccountMeter = metrics.NewRegisteredMeter("state/snapshot/generation/account/generated", nil) + snapRecoveredAccountMeter = metrics.NewRegisteredMeter("state/snapshot/generation/account/recovered", nil) + snapWipedAccountMeter = metrics.NewRegisteredMeter("state/snapshot/generation/account/wiped", nil) + snapMissallAccountMeter = metrics.NewRegisteredMeter("state/snapshot/generation/account/missall", nil) + snapGeneratedStorageMeter = metrics.NewRegisteredMeter("state/snapshot/generation/storage/generated", nil) + snapRecoveredStorageMeter = metrics.NewRegisteredMeter("state/snapshot/generation/storage/recovered", nil) + snapWipedStorageMeter = metrics.NewRegisteredMeter("state/snapshot/generation/storage/wiped", nil) + snapMissallStorageMeter = metrics.NewRegisteredMeter("state/snapshot/generation/storage/missall", nil) + snapDanglingStorageMeter = metrics.NewRegisteredMeter("state/snapshot/generation/storage/dangling", nil) + snapSuccessfulRangeProofMeter = metrics.NewRegisteredMeter("state/snapshot/generation/proof/success", nil) + snapFailedRangeProofMeter = metrics.NewRegisteredMeter("state/snapshot/generation/proof/failure", nil) + + // snapAccountProveCounter measures time spent on the account proving + snapAccountProveCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/account/prove", nil) + // snapAccountTrieReadCounter measures time spent on the account trie iteration + snapAccountTrieReadCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/account/trieread", nil) + // snapAccountSnapReadCounter measures time spent on the snapshot account iteration + snapAccountSnapReadCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/account/snapread", nil) + // snapAccountWriteCounter measures time spent on writing/updating/deleting accounts + snapAccountWriteCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/account/write", nil) + // snapStorageProveCounter measures time spent on storage proving + snapStorageProveCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/storage/prove", nil) + // snapStorageTrieReadCounter measures time spent on the storage trie iteration + snapStorageTrieReadCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/storage/trieread", nil) + // snapStorageSnapReadCounter measures time spent on the snapshot storage iteration + snapStorageSnapReadCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/storage/snapread", nil) + // snapStorageWriteCounter measures time spent on writing/updating storages + snapStorageWriteCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/storage/write", nil) + // snapStorageCleanCounter measures time spent on deleting storages + snapStorageCleanCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/storage/clean", nil) +) diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go new file mode 100644 index 0000000000..47dc1a3a17 --- /dev/null +++ b/core/state/snapshot/snapshot.go @@ -0,0 +1,854 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package snapshot implements a journalled, dynamic state dump. +package snapshot + +import ( + "bytes" + "errors" + "fmt" + "sync" + "sync/atomic" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/harmony-one/harmony/core/rawdb" + "github.com/harmony-one/harmony/internal/utils" +) + +var ( + snapshotCleanAccountHitMeter = metrics.NewRegisteredMeter("state/snapshot/clean/account/hit", nil) + snapshotCleanAccountMissMeter = metrics.NewRegisteredMeter("state/snapshot/clean/account/miss", nil) + snapshotCleanAccountInexMeter = metrics.NewRegisteredMeter("state/snapshot/clean/account/inex", nil) + snapshotCleanAccountReadMeter = metrics.NewRegisteredMeter("state/snapshot/clean/account/read", nil) + snapshotCleanAccountWriteMeter = metrics.NewRegisteredMeter("state/snapshot/clean/account/write", nil) + + snapshotCleanStorageHitMeter = metrics.NewRegisteredMeter("state/snapshot/clean/storage/hit", nil) + snapshotCleanStorageMissMeter = metrics.NewRegisteredMeter("state/snapshot/clean/storage/miss", nil) + snapshotCleanStorageInexMeter = metrics.NewRegisteredMeter("state/snapshot/clean/storage/inex", nil) + snapshotCleanStorageReadMeter = metrics.NewRegisteredMeter("state/snapshot/clean/storage/read", nil) + snapshotCleanStorageWriteMeter = metrics.NewRegisteredMeter("state/snapshot/clean/storage/write", nil) + + snapshotDirtyAccountHitMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/account/hit", nil) + snapshotDirtyAccountMissMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/account/miss", nil) + snapshotDirtyAccountInexMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/account/inex", nil) + snapshotDirtyAccountReadMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/account/read", nil) + snapshotDirtyAccountWriteMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/account/write", nil) + + snapshotDirtyStorageHitMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/storage/hit", nil) + snapshotDirtyStorageMissMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/storage/miss", nil) + snapshotDirtyStorageInexMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/storage/inex", nil) + snapshotDirtyStorageReadMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/storage/read", nil) + snapshotDirtyStorageWriteMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/storage/write", nil) + + snapshotDirtyAccountHitDepthHist = metrics.NewRegisteredHistogram("state/snapshot/dirty/account/hit/depth", nil, metrics.NewExpDecaySample(1028, 0.015)) + snapshotDirtyStorageHitDepthHist = metrics.NewRegisteredHistogram("state/snapshot/dirty/storage/hit/depth", nil, metrics.NewExpDecaySample(1028, 0.015)) + + snapshotFlushAccountItemMeter = metrics.NewRegisteredMeter("state/snapshot/flush/account/item", nil) + snapshotFlushAccountSizeMeter = metrics.NewRegisteredMeter("state/snapshot/flush/account/size", nil) + snapshotFlushStorageItemMeter = metrics.NewRegisteredMeter("state/snapshot/flush/storage/item", nil) + snapshotFlushStorageSizeMeter = metrics.NewRegisteredMeter("state/snapshot/flush/storage/size", nil) + + snapshotBloomIndexTimer = metrics.NewRegisteredResettingTimer("state/snapshot/bloom/index", nil) + snapshotBloomErrorGauge = metrics.NewRegisteredGaugeFloat64("state/snapshot/bloom/error", nil) + + snapshotBloomAccountTrueHitMeter = metrics.NewRegisteredMeter("state/snapshot/bloom/account/truehit", nil) + snapshotBloomAccountFalseHitMeter = metrics.NewRegisteredMeter("state/snapshot/bloom/account/falsehit", nil) + snapshotBloomAccountMissMeter = metrics.NewRegisteredMeter("state/snapshot/bloom/account/miss", nil) + + snapshotBloomStorageTrueHitMeter = metrics.NewRegisteredMeter("state/snapshot/bloom/storage/truehit", nil) + snapshotBloomStorageFalseHitMeter = metrics.NewRegisteredMeter("state/snapshot/bloom/storage/falsehit", nil) + snapshotBloomStorageMissMeter = metrics.NewRegisteredMeter("state/snapshot/bloom/storage/miss", nil) + + // ErrSnapshotStale is returned from data accessors if the underlying snapshot + // layer had been invalidated due to the chain progressing forward far enough + // to not maintain the layer's original state. + ErrSnapshotStale = errors.New("snapshot stale") + + // ErrNotCoveredYet is returned from data accessors if the underlying snapshot + // is being generated currently and the requested data item is not yet in the + // range of accounts covered. + ErrNotCoveredYet = errors.New("not covered yet") + + // ErrNotConstructed is returned if the callers want to iterate the snapshot + // while the generation is not finished yet. + ErrNotConstructed = errors.New("snapshot is not constructed") + + // errSnapshotCycle is returned if a snapshot is attempted to be inserted + // that forms a cycle in the snapshot tree. + errSnapshotCycle = errors.New("snapshot cycle") +) + +// Snapshot represents the functionality supported by a snapshot storage layer. +type Snapshot interface { + // Root returns the root hash for which this snapshot was made. + Root() common.Hash + + // Account directly retrieves the account associated with a particular hash in + // the snapshot slim data format. + Account(hash common.Hash) (*Account, error) + + // AccountRLP directly retrieves the account RLP associated with a particular + // hash in the snapshot slim data format. + AccountRLP(hash common.Hash) ([]byte, error) + + // Storage directly retrieves the storage data associated with a particular hash, + // within a particular account. + Storage(accountHash, storageHash common.Hash) ([]byte, error) +} + +// snapshot is the internal version of the snapshot data layer that supports some +// additional methods compared to the public API. +type snapshot interface { + Snapshot + + // Parent returns the subsequent layer of a snapshot, or nil if the base was + // reached. + // + // Note, the method is an internal helper to avoid type switching between the + // disk and diff layers. There is no locking involved. + Parent() snapshot + + // Update creates a new layer on top of the existing snapshot diff tree with + // the specified data items. + // + // Note, the maps are retained by the method to avoid copying everything. + Update(blockRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer + + // Journal commits an entire diff hierarchy to disk into a single journal entry. + // This is meant to be used during shutdown to persist the snapshot without + // flattening everything down (bad for reorgs). + Journal(buffer *bytes.Buffer) (common.Hash, error) + + // Stale return whether this layer has become stale (was flattened across) or + // if it's still live. + Stale() bool + + // AccountIterator creates an account iterator over an arbitrary layer. + AccountIterator(seek common.Hash) AccountIterator + + // StorageIterator creates a storage iterator over an arbitrary layer. + StorageIterator(account common.Hash, seek common.Hash) (StorageIterator, bool) +} + +// Config includes the configurations for snapshots. +type Config struct { + CacheSize int // Megabytes permitted to use for read caches + Recovery bool // Indicator that the snapshots is in the recovery mode + NoBuild bool // Indicator that the snapshots generation is disallowed + AsyncBuild bool // The snapshot generation is allowed to be constructed asynchronously +} + +// Tree is an Ethereum state snapshot tree. It consists of one persistent base +// layer backed by a key-value store, on top of which arbitrarily many in-memory +// diff layers are topped. The memory diffs can form a tree with branching, but +// the disk layer is singleton and common to all. If a reorg goes deeper than the +// disk layer, everything needs to be deleted. +// +// The goal of a state snapshot is twofold: to allow direct access to account and +// storage data to avoid expensive multi-level trie lookups; and to allow sorted, +// cheap iteration of the account/storage tries for sync aid. +type Tree struct { + config Config // Snapshots configurations + diskdb ethdb.KeyValueStore // Persistent database to store the snapshot + triedb *trie.Database // In-memory cache to access the trie through + layers map[common.Hash]snapshot // Collection of all known layers + lock sync.RWMutex + + // Test hooks + onFlatten func() // Hook invoked when the bottom most diff layers are flattened +} + +// New attempts to load an already existing snapshot from a persistent key-value +// store (with a number of memory layers from a journal), ensuring that the head +// of the snapshot matches the expected one. +// +// If the snapshot is missing or the disk layer is broken, the snapshot will be +// reconstructed using both the existing data and the state trie. +// The repair happens on a background thread. +// +// If the memory layers in the journal do not match the disk layer (e.g. there is +// a gap) or the journal is missing, there are two repair cases: +// +// - if the 'recovery' parameter is true, memory diff-layers and the disk-layer +// will all be kept. This case happens when the snapshot is 'ahead' of the +// state trie. +// - otherwise, the entire snapshot is considered invalid and will be recreated on +// a background thread. +func New(config Config, diskdb ethdb.KeyValueStore, triedb *trie.Database, root common.Hash) (*Tree, error) { + // Create a new, empty snapshot tree + snap := &Tree{ + config: config, + diskdb: diskdb, + triedb: triedb, + layers: make(map[common.Hash]snapshot), + } + // Attempt to load a previously persisted snapshot and rebuild one if failed + head, disabled, err := loadSnapshot(diskdb, triedb, root, config.CacheSize, config.Recovery, config.NoBuild) + if disabled { + utils.Logger().Warn().Err(err).Msg("Snapshot maintenance disabled (syncing)") + return snap, nil + } + // Create the building waiter iff the background generation is allowed + if !config.NoBuild && !config.AsyncBuild { + defer snap.waitBuild() + } + if err != nil { + utils.Logger().Warn().Err(err).Msg("Failed to load snapshot") + if !config.NoBuild { + snap.Rebuild(root) + return snap, nil + } + return nil, err // Bail out the error, don't rebuild automatically. + } + // Existing snapshot loaded, seed all the layers + for head != nil { + snap.layers[head.Root()] = head + head = head.Parent() + } + return snap, nil +} + +// waitBuild blocks until the snapshot finishes rebuilding. This method is meant +// to be used by tests to ensure we're testing what we believe we are. +func (t *Tree) waitBuild() { + // Find the rebuild termination channel + var done chan struct{} + + t.lock.RLock() + for _, layer := range t.layers { + if layer, ok := layer.(*diskLayer); ok { + done = layer.genPending + break + } + } + t.lock.RUnlock() + + // Wait until the snapshot is generated + if done != nil { + <-done + } +} + +// Disable interrupts any pending snapshot generator, deletes all the snapshot +// layers in memory and marks snapshots disabled globally. In order to resume +// the snapshot functionality, the caller must invoke Rebuild. +func (t *Tree) Disable() { + // Interrupt any live snapshot layers + t.lock.Lock() + defer t.lock.Unlock() + + for _, layer := range t.layers { + switch layer := layer.(type) { + case *diskLayer: + // If the base layer is generating, abort it + if layer.genAbort != nil { + abort := make(chan *generatorStats) + layer.genAbort <- abort + <-abort + } + // Layer should be inactive now, mark it as stale + layer.lock.Lock() + layer.stale = true + layer.lock.Unlock() + + case *diffLayer: + // If the layer is a simple diff, simply mark as stale + layer.lock.Lock() + atomic.StoreUint32(&layer.stale, 1) + layer.lock.Unlock() + + default: + panic(fmt.Sprintf("unknown layer type: %T", layer)) + } + } + t.layers = map[common.Hash]snapshot{} + + // Delete all snapshot liveness information from the database + batch := t.diskdb.NewBatch() + + rawdb.WriteSnapshotDisabled(batch) + rawdb.DeleteSnapshotRoot(batch) + rawdb.DeleteSnapshotJournal(batch) + rawdb.DeleteSnapshotGenerator(batch) + rawdb.DeleteSnapshotRecoveryNumber(batch) + // Note, we don't delete the sync progress + + if err := batch.Write(); err != nil { + utils.Logger().Fatal().Err(err).Msg("Failed to disable snapshots") + } +} + +// Snapshot retrieves a snapshot belonging to the given block root, or nil if no +// snapshot is maintained for that block. +func (t *Tree) Snapshot(blockRoot common.Hash) Snapshot { + t.lock.RLock() + defer t.lock.RUnlock() + + return t.layers[blockRoot] +} + +// Snapshots returns all visited layers from the topmost layer with specific +// root and traverses downward. The layer amount is limited by the given number. +// If nodisk is set, then disk layer is excluded. +func (t *Tree) Snapshots(root common.Hash, limits int, nodisk bool) []Snapshot { + t.lock.RLock() + defer t.lock.RUnlock() + + if limits == 0 { + return nil + } + layer := t.layers[root] + if layer == nil { + return nil + } + var ret []Snapshot + for { + if _, isdisk := layer.(*diskLayer); isdisk && nodisk { + break + } + ret = append(ret, layer) + limits -= 1 + if limits == 0 { + break + } + parent := layer.Parent() + if parent == nil { + break + } + layer = parent + } + return ret +} + +// Update adds a new snapshot into the tree, if that can be linked to an existing +// old parent. It is disallowed to insert a disk layer (the origin of all). +func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + // Reject noop updates to avoid self-loops in the snapshot tree. This is a + // special case that can only happen for Clique networks where empty blocks + // don't modify the state (0 block subsidy). + // + // Although we could silently ignore this internally, it should be the caller's + // responsibility to avoid even attempting to insert such a snapshot. + if blockRoot == parentRoot { + return errSnapshotCycle + } + // Generate a new snapshot on top of the parent + parent := t.Snapshot(parentRoot) + if parent == nil { + return fmt.Errorf("parent [%#x] snapshot missing", parentRoot) + } + snap := parent.(snapshot).Update(blockRoot, destructs, accounts, storage) + + // Save the new snapshot for later + t.lock.Lock() + defer t.lock.Unlock() + + t.layers[snap.root] = snap + return nil +} + +// Cap traverses downwards the snapshot tree from a head block hash until the +// number of allowed layers are crossed. All layers beyond the permitted number +// are flattened downwards. +// +// Note, the final diff layer count in general will be one more than the amount +// requested. This happens because the bottom-most diff layer is the accumulator +// which may or may not overflow and cascade to disk. Since this last layer's +// survival is only known *after* capping, we need to omit it from the count if +// we want to ensure that *at least* the requested number of diff layers remain. +func (t *Tree) Cap(root common.Hash, layers int) error { + // Retrieve the head snapshot to cap from + snap := t.Snapshot(root) + if snap == nil { + return fmt.Errorf("snapshot [%#x] missing", root) + } + diff, ok := snap.(*diffLayer) + if !ok { + return fmt.Errorf("snapshot [%#x] is disk layer", root) + } + // If the generator is still running, use a more aggressive cap + diff.origin.lock.RLock() + if diff.origin.genMarker != nil && layers > 8 { + layers = 8 + } + diff.origin.lock.RUnlock() + + // Run the internal capping and discard all stale layers + t.lock.Lock() + defer t.lock.Unlock() + + // Flattening the bottom-most diff layer requires special casing since there's + // no child to rewire to the grandparent. In that case we can fake a temporary + // child for the capping and then remove it. + if layers == 0 { + // If full commit was requested, flatten the diffs and merge onto disk + diff.lock.RLock() + base := diffToDisk(diff.flatten().(*diffLayer)) + diff.lock.RUnlock() + + // Replace the entire snapshot tree with the flat base + t.layers = map[common.Hash]snapshot{base.root: base} + return nil + } + persisted := t.cap(diff, layers) + + // Remove any layer that is stale or links into a stale layer + children := make(map[common.Hash][]common.Hash) + for root, snap := range t.layers { + if diff, ok := snap.(*diffLayer); ok { + parent := diff.parent.Root() + children[parent] = append(children[parent], root) + } + } + var remove func(root common.Hash) + remove = func(root common.Hash) { + delete(t.layers, root) + for _, child := range children[root] { + remove(child) + } + delete(children, root) + } + for root, snap := range t.layers { + if snap.Stale() { + remove(root) + } + } + // If the disk layer was modified, regenerate all the cumulative blooms + if persisted != nil { + var rebloom func(root common.Hash) + rebloom = func(root common.Hash) { + if diff, ok := t.layers[root].(*diffLayer); ok { + diff.rebloom(persisted) + } + for _, child := range children[root] { + rebloom(child) + } + } + rebloom(persisted.root) + } + return nil +} + +// cap traverses downwards the diff tree until the number of allowed layers are +// crossed. All diffs beyond the permitted number are flattened downwards. If the +// layer limit is reached, memory cap is also enforced (but not before). +// +// The method returns the new disk layer if diffs were persisted into it. +// +// Note, the final diff layer count in general will be one more than the amount +// requested. This happens because the bottom-most diff layer is the accumulator +// which may or may not overflow and cascade to disk. Since this last layer's +// survival is only known *after* capping, we need to omit it from the count if +// we want to ensure that *at least* the requested number of diff layers remain. +func (t *Tree) cap(diff *diffLayer, layers int) *diskLayer { + // Dive until we run out of layers or reach the persistent database + for i := 0; i < layers-1; i++ { + // If we still have diff layers below, continue down + if parent, ok := diff.parent.(*diffLayer); ok { + diff = parent + } else { + // Diff stack too shallow, return without modifications + return nil + } + } + // We're out of layers, flatten anything below, stopping if it's the disk or if + // the memory limit is not yet exceeded. + switch parent := diff.parent.(type) { + case *diskLayer: + return nil + + case *diffLayer: + // Hold the write lock until the flattened parent is linked correctly. + // Otherwise, the stale layer may be accessed by external reads in the + // meantime. + diff.lock.Lock() + defer diff.lock.Unlock() + + // Flatten the parent into the grandparent. The flattening internally obtains a + // write lock on grandparent. + flattened := parent.flatten().(*diffLayer) + t.layers[flattened.root] = flattened + + // Invoke the hook if it's registered. Ugly hack. + if t.onFlatten != nil { + t.onFlatten() + } + diff.parent = flattened + if flattened.memory < aggregatorMemoryLimit { + // Accumulator layer is smaller than the limit, so we can abort, unless + // there's a snapshot being generated currently. In that case, the trie + // will move from underneath the generator so we **must** merge all the + // partial data down into the snapshot and restart the generation. + if flattened.parent.(*diskLayer).genAbort == nil { + return nil + } + } + default: + panic(fmt.Sprintf("unknown data layer: %T", parent)) + } + // If the bottom-most layer is larger than our memory cap, persist to disk + bottom := diff.parent.(*diffLayer) + + bottom.lock.RLock() + base := diffToDisk(bottom) + bottom.lock.RUnlock() + + t.layers[base.root] = base + diff.parent = base + return base +} + +// diffToDisk merges a bottom-most diff into the persistent disk layer underneath +// it. The method will panic if called onto a non-bottom-most diff layer. +// +// The disk layer persistence should be operated in an atomic way. All updates should +// be discarded if the whole transition if not finished. +func diffToDisk(bottom *diffLayer) *diskLayer { + var ( + base = bottom.parent.(*diskLayer) + batch = base.diskdb.NewBatch() + stats *generatorStats + ) + // If the disk layer is running a snapshot generator, abort it + if base.genAbort != nil { + abort := make(chan *generatorStats) + base.genAbort <- abort + stats = <-abort + } + // Put the deletion in the batch writer, flush all updates in the final step. + rawdb.DeleteSnapshotRoot(batch) + + // Mark the original base as stale as we're going to create a new wrapper + base.lock.Lock() + if base.stale { + panic("parent disk layer is stale") // we've committed into the same base from two children, boo + } + base.stale = true + base.lock.Unlock() + + // Destroy all the destructed accounts from the database + for hash := range bottom.destructSet { + // Skip any account not covered yet by the snapshot + if base.genMarker != nil && bytes.Compare(hash[:], base.genMarker) > 0 { + continue + } + // Remove all storage slots + rawdb.DeleteAccountSnapshot(batch, hash) + base.cache.Set(hash[:], nil) + + it := rawdb.IterateStorageSnapshots(base.diskdb, hash) + for it.Next() { + key := it.Key() + batch.Delete(key) + base.cache.Del(key[1:]) + snapshotFlushStorageItemMeter.Mark(1) + + // Ensure we don't delete too much data blindly (contract can be + // huge). It's ok to flush, the root will go missing in case of a + // crash and we'll detect and regenerate the snapshot. + if batch.ValueSize() > ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + utils.Logger().Fatal().Err(err).Msg("Failed to write storage deletions") + } + batch.Reset() + } + } + it.Release() + } + // Push all updated accounts into the database + for hash, data := range bottom.accountData { + // Skip any account not covered yet by the snapshot + if base.genMarker != nil && bytes.Compare(hash[:], base.genMarker) > 0 { + continue + } + // Push the account to disk + rawdb.WriteAccountSnapshot(batch, hash, data) + base.cache.Set(hash[:], data) + snapshotCleanAccountWriteMeter.Mark(int64(len(data))) + + snapshotFlushAccountItemMeter.Mark(1) + snapshotFlushAccountSizeMeter.Mark(int64(len(data))) + + // Ensure we don't write too much data blindly. It's ok to flush, the + // root will go missing in case of a crash and we'll detect and regen + // the snapshot. + if batch.ValueSize() > ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + utils.Logger().Fatal().Err(err).Msg("Failed to write storage deletions") + } + batch.Reset() + } + } + // Push all the storage slots into the database + for accountHash, storage := range bottom.storageData { + // Skip any account not covered yet by the snapshot + if base.genMarker != nil && bytes.Compare(accountHash[:], base.genMarker) > 0 { + continue + } + // Generation might be mid-account, track that case too + midAccount := base.genMarker != nil && bytes.Equal(accountHash[:], base.genMarker[:common.HashLength]) + + for storageHash, data := range storage { + // Skip any slot not covered yet by the snapshot + if midAccount && bytes.Compare(storageHash[:], base.genMarker[common.HashLength:]) > 0 { + continue + } + if len(data) > 0 { + rawdb.WriteStorageSnapshot(batch, accountHash, storageHash, data) + base.cache.Set(append(accountHash[:], storageHash[:]...), data) + snapshotCleanStorageWriteMeter.Mark(int64(len(data))) + } else { + rawdb.DeleteStorageSnapshot(batch, accountHash, storageHash) + base.cache.Set(append(accountHash[:], storageHash[:]...), nil) + } + snapshotFlushStorageItemMeter.Mark(1) + snapshotFlushStorageSizeMeter.Mark(int64(len(data))) + } + } + // Update the snapshot block marker and write any remainder data + rawdb.WriteSnapshotRoot(batch, bottom.root) + + // Write out the generator progress marker and report + journalProgress(batch, base.genMarker, stats) + + // Flush all the updates in the single db operation. Ensure the + // disk layer transition is atomic. + if err := batch.Write(); err != nil { + utils.Logger().Fatal().Err(err).Msg("Failed to write leftover snapshot") + } + utils.Logger().Debug().Interface("root", bottom.root).Bool("complete", base.genMarker == nil).Msg("Journalled disk layer") + res := &diskLayer{ + root: bottom.root, + cache: base.cache, + diskdb: base.diskdb, + triedb: base.triedb, + genMarker: base.genMarker, + genPending: base.genPending, + } + // If snapshot generation hasn't finished yet, port over all the starts and + // continue where the previous round left off. + // + // Note, the `base.genAbort` comparison is not used normally, it's checked + // to allow the tests to play with the marker without triggering this path. + if base.genMarker != nil && base.genAbort != nil { + res.genMarker = base.genMarker + res.genAbort = make(chan chan *generatorStats) + go res.generate(stats) + } + return res +} + +// Journal commits an entire diff hierarchy to disk into a single journal entry. +// This is meant to be used during shutdown to persist the snapshot without +// flattening everything down (bad for reorgs). +// +// The method returns the root hash of the base layer that needs to be persisted +// to disk as a trie too to allow continuing any pending generation op. +func (t *Tree) Journal(root common.Hash) (common.Hash, error) { + // Retrieve the head snapshot to journal from var snap snapshot + snap := t.Snapshot(root) + if snap == nil { + return common.Hash{}, fmt.Errorf("snapshot [%#x] missing", root) + } + // Run the journaling + t.lock.Lock() + defer t.lock.Unlock() + + // Firstly write out the metadata of journal + journal := new(bytes.Buffer) + if err := rlp.Encode(journal, journalVersion); err != nil { + return common.Hash{}, err + } + diskroot := t.diskRoot() + if diskroot == (common.Hash{}) { + return common.Hash{}, errors.New("invalid disk root") + } + // Secondly write out the disk layer root, ensure the + // diff journal is continuous with disk. + if err := rlp.Encode(journal, diskroot); err != nil { + return common.Hash{}, err + } + // Finally write out the journal of each layer in reverse order. + base, err := snap.(snapshot).Journal(journal) + if err != nil { + return common.Hash{}, err + } + // Store the journal into the database and return + rawdb.WriteSnapshotJournal(t.diskdb, journal.Bytes()) + return base, nil +} + +// Rebuild wipes all available snapshot data from the persistent database and +// discard all caches and diff layers. Afterwards, it starts a new snapshot +// generator with the given root hash. +func (t *Tree) Rebuild(root common.Hash) { + t.lock.Lock() + defer t.lock.Unlock() + + // Firstly delete any recovery flag in the database. Because now we are + // building a brand new snapshot. Also reenable the snapshot feature. + rawdb.DeleteSnapshotRecoveryNumber(t.diskdb) + rawdb.DeleteSnapshotDisabled(t.diskdb) + + // Iterate over and mark all layers stale + for _, layer := range t.layers { + switch layer := layer.(type) { + case *diskLayer: + // If the base layer is generating, abort it and save + if layer.genAbort != nil { + abort := make(chan *generatorStats) + layer.genAbort <- abort + <-abort + } + // Layer should be inactive now, mark it as stale + layer.lock.Lock() + layer.stale = true + layer.lock.Unlock() + + case *diffLayer: + // If the layer is a simple diff, simply mark as stale + layer.lock.Lock() + atomic.StoreUint32(&layer.stale, 1) + layer.lock.Unlock() + + default: + panic(fmt.Sprintf("unknown layer type: %T", layer)) + } + } + // Start generating a new snapshot from scratch on a background thread. The + // generator will run a wiper first if there's not one running right now. + utils.Logger().Info().Msg("Rebuilding state snapshot") + t.layers = map[common.Hash]snapshot{ + root: generateSnapshot(t.diskdb, t.triedb, t.config.CacheSize, root), + } +} + +// AccountIterator creates a new account iterator for the specified root hash and +// seeks to a starting account hash. +func (t *Tree) AccountIterator(root common.Hash, seek common.Hash) (AccountIterator, error) { + ok, err := t.generating() + if err != nil { + return nil, err + } + if ok { + return nil, ErrNotConstructed + } + return newFastAccountIterator(t, root, seek) +} + +// StorageIterator creates a new storage iterator for the specified root hash and +// account. The iterator will be move to the specific start position. +func (t *Tree) StorageIterator(root common.Hash, account common.Hash, seek common.Hash) (StorageIterator, error) { + ok, err := t.generating() + if err != nil { + return nil, err + } + if ok { + return nil, ErrNotConstructed + } + return newFastStorageIterator(t, root, account, seek) +} + +// Verify iterates the whole state(all the accounts as well as the corresponding storages) +// with the specific root and compares the re-computed hash with the original one. +func (t *Tree) Verify(root common.Hash) error { + acctIt, err := t.AccountIterator(root, common.Hash{}) + if err != nil { + return err + } + defer acctIt.Release() + + got, err := generateTrieRoot(nil, "", acctIt, common.Hash{}, stackTrieGenerate, func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) { + storageIt, err := t.StorageIterator(root, accountHash, common.Hash{}) + if err != nil { + return common.Hash{}, err + } + defer storageIt.Release() + + hash, err := generateTrieRoot(nil, "", storageIt, accountHash, stackTrieGenerate, nil, stat, false) + if err != nil { + return common.Hash{}, err + } + return hash, nil + }, newGenerateStats(), true) + + if err != nil { + return err + } + if got != root { + return fmt.Errorf("state root hash mismatch: got %x, want %x", got, root) + } + return nil +} + +// disklayer is an internal helper function to return the disk layer. +// The lock of snapTree is assumed to be held already. +func (t *Tree) disklayer() *diskLayer { + var snap snapshot + for _, s := range t.layers { + snap = s + break + } + if snap == nil { + return nil + } + switch layer := snap.(type) { + case *diskLayer: + return layer + case *diffLayer: + return layer.origin + default: + panic(fmt.Sprintf("%T: undefined layer", snap)) + } +} + +// diskRoot is a internal helper function to return the disk layer root. +// The lock of snapTree is assumed to be held already. +func (t *Tree) diskRoot() common.Hash { + disklayer := t.disklayer() + if disklayer == nil { + return common.Hash{} + } + return disklayer.Root() +} + +// generating is an internal helper function which reports whether the snapshot +// is still under the construction. +func (t *Tree) generating() (bool, error) { + t.lock.Lock() + defer t.lock.Unlock() + + layer := t.disklayer() + if layer == nil { + return false, errors.New("disk layer is missing") + } + layer.lock.RLock() + defer layer.lock.RUnlock() + return layer.genMarker != nil, nil +} + +// DiskRoot is a external helper function to return the disk layer root. +func (t *Tree) DiskRoot() common.Hash { + t.lock.Lock() + defer t.lock.Unlock() + + return t.diskRoot() +} diff --git a/core/state/snapshot/snapshot_test.go b/core/state/snapshot/snapshot_test.go new file mode 100644 index 0000000000..951576bc8f --- /dev/null +++ b/core/state/snapshot/snapshot_test.go @@ -0,0 +1,488 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + crand "crypto/rand" + "encoding/binary" + "fmt" + "math/big" + "math/rand" + "testing" + "time" + + "github.com/VictoriaMetrics/fastcache" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/harmony-one/harmony/core/rawdb" +) + +// randomHash generates a random blob of data and returns it as a hash. +func randomHash() common.Hash { + var hash common.Hash + if n, err := crand.Read(hash[:]); n != common.HashLength || err != nil { + panic(err) + } + return hash +} + +// randomAccount generates a random account and returns it RLP encoded. +func randomAccount() []byte { + root := randomHash() + a := Account{ + Balance: big.NewInt(rand.Int63()), + Nonce: rand.Uint64(), + Root: root[:], + CodeHash: types.EmptyCodeHash[:], + } + data, _ := rlp.EncodeToBytes(a) + return data +} + +// randomAccountSet generates a set of random accounts with the given strings as +// the account address hashes. +func randomAccountSet(hashes ...string) map[common.Hash][]byte { + accounts := make(map[common.Hash][]byte) + for _, hash := range hashes { + accounts[common.HexToHash(hash)] = randomAccount() + } + return accounts +} + +// randomStorageSet generates a set of random slots with the given strings as +// the slot addresses. +func randomStorageSet(accounts []string, hashes [][]string, nilStorage [][]string) map[common.Hash]map[common.Hash][]byte { + storages := make(map[common.Hash]map[common.Hash][]byte) + for index, account := range accounts { + storages[common.HexToHash(account)] = make(map[common.Hash][]byte) + + if index < len(hashes) { + hashes := hashes[index] + for _, hash := range hashes { + storages[common.HexToHash(account)][common.HexToHash(hash)] = randomHash().Bytes() + } + } + if index < len(nilStorage) { + nils := nilStorage[index] + for _, hash := range nils { + storages[common.HexToHash(account)][common.HexToHash(hash)] = nil + } + } + } + return storages +} + +// Tests that if a disk layer becomes stale, no active external references will +// be returned with junk data. This version of the test flattens every diff layer +// to check internal corner case around the bottom-most memory accumulator. +func TestDiskLayerExternalInvalidationFullFlatten(t *testing.T) { + // Create an empty base layer and a snapshot tree out of it + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + // Retrieve a reference to the base and commit a diff on top + ref := snaps.Snapshot(base.root) + + accounts := map[common.Hash][]byte{ + common.HexToHash("0xa1"): randomAccount(), + } + if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { + t.Fatalf("failed to create a diff layer: %v", err) + } + if n := len(snaps.layers); n != 2 { + t.Errorf("pre-cap layer count mismatch: have %d, want %d", n, 2) + } + // Commit the diff layer onto the disk and ensure it's persisted + if err := snaps.Cap(common.HexToHash("0x02"), 0); err != nil { + t.Fatalf("failed to merge diff layer onto disk: %v", err) + } + // Since the base layer was modified, ensure that data retrieval on the external reference fail + if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale { + t.Errorf("stale reference returned account: %#x (err: %v)", acc, err) + } + if slot, err := ref.Storage(common.HexToHash("0xa1"), common.HexToHash("0xb1")); err != ErrSnapshotStale { + t.Errorf("stale reference returned storage slot: %#x (err: %v)", slot, err) + } + if n := len(snaps.layers); n != 1 { + t.Errorf("post-cap layer count mismatch: have %d, want %d", n, 1) + fmt.Println(snaps.layers) + } +} + +// Tests that if a disk layer becomes stale, no active external references will +// be returned with junk data. This version of the test retains the bottom diff +// layer to check the usual mode of operation where the accumulator is retained. +func TestDiskLayerExternalInvalidationPartialFlatten(t *testing.T) { + // Create an empty base layer and a snapshot tree out of it + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + // Retrieve a reference to the base and commit two diffs on top + ref := snaps.Snapshot(base.root) + + accounts := map[common.Hash][]byte{ + common.HexToHash("0xa1"): randomAccount(), + } + if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { + t.Fatalf("failed to create a diff layer: %v", err) + } + if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil { + t.Fatalf("failed to create a diff layer: %v", err) + } + if n := len(snaps.layers); n != 3 { + t.Errorf("pre-cap layer count mismatch: have %d, want %d", n, 3) + } + // Commit the diff layer onto the disk and ensure it's persisted + defer func(memcap uint64) { aggregatorMemoryLimit = memcap }(aggregatorMemoryLimit) + aggregatorMemoryLimit = 0 + + if err := snaps.Cap(common.HexToHash("0x03"), 1); err != nil { + t.Fatalf("failed to merge accumulator onto disk: %v", err) + } + // Since the base layer was modified, ensure that data retrievals on the external reference fail + if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale { + t.Errorf("stale reference returned account: %#x (err: %v)", acc, err) + } + if slot, err := ref.Storage(common.HexToHash("0xa1"), common.HexToHash("0xb1")); err != ErrSnapshotStale { + t.Errorf("stale reference returned storage slot: %#x (err: %v)", slot, err) + } + if n := len(snaps.layers); n != 2 { + t.Errorf("post-cap layer count mismatch: have %d, want %d", n, 2) + fmt.Println(snaps.layers) + } +} + +// Tests that if a diff layer becomes stale, no active external references will +// be returned with junk data. This version of the test retains the bottom diff +// layer to check the usual mode of operation where the accumulator is retained. +func TestDiffLayerExternalInvalidationPartialFlatten(t *testing.T) { + // Create an empty base layer and a snapshot tree out of it + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + // Commit three diffs on top and retrieve a reference to the bottommost + accounts := map[common.Hash][]byte{ + common.HexToHash("0xa1"): randomAccount(), + } + if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { + t.Fatalf("failed to create a diff layer: %v", err) + } + if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil { + t.Fatalf("failed to create a diff layer: %v", err) + } + if err := snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, accounts, nil); err != nil { + t.Fatalf("failed to create a diff layer: %v", err) + } + if n := len(snaps.layers); n != 4 { + t.Errorf("pre-cap layer count mismatch: have %d, want %d", n, 4) + } + ref := snaps.Snapshot(common.HexToHash("0x02")) + + // Doing a Cap operation with many allowed layers should be a no-op + exp := len(snaps.layers) + if err := snaps.Cap(common.HexToHash("0x04"), 2000); err != nil { + t.Fatalf("failed to flatten diff layer into accumulator: %v", err) + } + if got := len(snaps.layers); got != exp { + t.Errorf("layers modified, got %d exp %d", got, exp) + } + // Flatten the diff layer into the bottom accumulator + if err := snaps.Cap(common.HexToHash("0x04"), 1); err != nil { + t.Fatalf("failed to flatten diff layer into accumulator: %v", err) + } + // Since the accumulator diff layer was modified, ensure that data retrievals on the external reference fail + if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale { + t.Errorf("stale reference returned account: %#x (err: %v)", acc, err) + } + if slot, err := ref.Storage(common.HexToHash("0xa1"), common.HexToHash("0xb1")); err != ErrSnapshotStale { + t.Errorf("stale reference returned storage slot: %#x (err: %v)", slot, err) + } + if n := len(snaps.layers); n != 3 { + t.Errorf("post-cap layer count mismatch: have %d, want %d", n, 3) + fmt.Println(snaps.layers) + } +} + +// TestPostCapBasicDataAccess tests some functionality regarding capping/flattening. +func TestPostCapBasicDataAccess(t *testing.T) { + // setAccount is a helper to construct a random account entry and assign it to + // an account slot in a snapshot + setAccount := func(accKey string) map[common.Hash][]byte { + return map[common.Hash][]byte{ + common.HexToHash(accKey): randomAccount(), + } + } + // Create a starting base layer and a snapshot tree out of it + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + // The lowest difflayer + snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), nil, setAccount("0xa1"), nil) + snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), nil, setAccount("0xa2"), nil) + snaps.Update(common.HexToHash("0xb2"), common.HexToHash("0xa1"), nil, setAccount("0xb2"), nil) + + snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil) + snaps.Update(common.HexToHash("0xb3"), common.HexToHash("0xb2"), nil, setAccount("0xb3"), nil) + + // checkExist verifies if an account exists in a snapshot + checkExist := func(layer *diffLayer, key string) error { + if data, _ := layer.Account(common.HexToHash(key)); data == nil { + return fmt.Errorf("expected %x to exist, got nil", common.HexToHash(key)) + } + return nil + } + // shouldErr checks that an account access errors as expected + shouldErr := func(layer *diffLayer, key string) error { + if data, err := layer.Account(common.HexToHash(key)); err == nil { + return fmt.Errorf("expected error, got data %x", data) + } + return nil + } + // check basics + snap := snaps.Snapshot(common.HexToHash("0xb3")).(*diffLayer) + + if err := checkExist(snap, "0xa1"); err != nil { + t.Error(err) + } + if err := checkExist(snap, "0xb2"); err != nil { + t.Error(err) + } + if err := checkExist(snap, "0xb3"); err != nil { + t.Error(err) + } + // Cap to a bad root should fail + if err := snaps.Cap(common.HexToHash("0x1337"), 0); err == nil { + t.Errorf("expected error, got none") + } + // Now, merge the a-chain + snaps.Cap(common.HexToHash("0xa3"), 0) + + // At this point, a2 got merged into a1. Thus, a1 is now modified, and as a1 is + // the parent of b2, b2 should no longer be able to iterate into parent. + + // These should still be accessible + if err := checkExist(snap, "0xb2"); err != nil { + t.Error(err) + } + if err := checkExist(snap, "0xb3"); err != nil { + t.Error(err) + } + // But these would need iteration into the modified parent + if err := shouldErr(snap, "0xa1"); err != nil { + t.Error(err) + } + if err := shouldErr(snap, "0xa2"); err != nil { + t.Error(err) + } + if err := shouldErr(snap, "0xa3"); err != nil { + t.Error(err) + } + // Now, merge it again, just for fun. It should now error, since a3 + // is a disk layer + if err := snaps.Cap(common.HexToHash("0xa3"), 0); err == nil { + t.Error("expected error capping the disk layer, got none") + } +} + +// TestSnaphots tests the functionality for retrieving the snapshot +// with given head root and the desired depth. +func TestSnaphots(t *testing.T) { + // setAccount is a helper to construct a random account entry and assign it to + // an account slot in a snapshot + setAccount := func(accKey string) map[common.Hash][]byte { + return map[common.Hash][]byte{ + common.HexToHash(accKey): randomAccount(), + } + } + makeRoot := func(height uint64) common.Hash { + var buffer [8]byte + binary.BigEndian.PutUint64(buffer[:], height) + return common.BytesToHash(buffer[:]) + } + // Create a starting base layer and a snapshot tree out of it + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: makeRoot(1), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + // Construct the snapshots with 129 layers, flattening whatever's above that + var ( + last = common.HexToHash("0x01") + head common.Hash + ) + for i := 0; i < 129; i++ { + head = makeRoot(uint64(i + 2)) + snaps.Update(head, last, nil, setAccount(fmt.Sprintf("%d", i+2)), nil) + last = head + snaps.Cap(head, 128) // 130 layers (128 diffs + 1 accumulator + 1 disk) + } + var cases = []struct { + headRoot common.Hash + limit int + nodisk bool + expected int + expectBottom common.Hash + }{ + {head, 0, false, 0, common.Hash{}}, + {head, 64, false, 64, makeRoot(129 + 2 - 64)}, + {head, 128, false, 128, makeRoot(3)}, // Normal diff layers, no accumulator + {head, 129, true, 129, makeRoot(2)}, // All diff layers, including accumulator + {head, 130, false, 130, makeRoot(1)}, // All diff layers + disk layer + } + for i, c := range cases { + layers := snaps.Snapshots(c.headRoot, c.limit, c.nodisk) + if len(layers) != c.expected { + t.Errorf("non-overflow test %d: returned snapshot layers are mismatched, want %v, got %v", i, c.expected, len(layers)) + } + if len(layers) == 0 { + continue + } + bottommost := layers[len(layers)-1] + if bottommost.Root() != c.expectBottom { + t.Errorf("non-overflow test %d: snapshot mismatch, want %v, get %v", i, c.expectBottom, bottommost.Root()) + } + } + // Above we've tested the normal capping, which leaves the accumulator live. + // Test that if the bottommost accumulator diff layer overflows the allowed + // memory limit, the snapshot tree gets capped to one less layer. + // Commit the diff layer onto the disk and ensure it's persisted + defer func(memcap uint64) { aggregatorMemoryLimit = memcap }(aggregatorMemoryLimit) + aggregatorMemoryLimit = 0 + + snaps.Cap(head, 128) // 129 (128 diffs + 1 overflown accumulator + 1 disk) + + cases = []struct { + headRoot common.Hash + limit int + nodisk bool + expected int + expectBottom common.Hash + }{ + {head, 0, false, 0, common.Hash{}}, + {head, 64, false, 64, makeRoot(129 + 2 - 64)}, + {head, 128, false, 128, makeRoot(3)}, // All diff layers, accumulator was flattened + {head, 129, true, 128, makeRoot(3)}, // All diff layers, accumulator was flattened + {head, 130, false, 129, makeRoot(2)}, // All diff layers + disk layer + } + for i, c := range cases { + layers := snaps.Snapshots(c.headRoot, c.limit, c.nodisk) + if len(layers) != c.expected { + t.Errorf("overflow test %d: returned snapshot layers are mismatched, want %v, got %v", i, c.expected, len(layers)) + } + if len(layers) == 0 { + continue + } + bottommost := layers[len(layers)-1] + if bottommost.Root() != c.expectBottom { + t.Errorf("overflow test %d: snapshot mismatch, want %v, get %v", i, c.expectBottom, bottommost.Root()) + } + } +} + +// TestReadStateDuringFlattening tests the scenario that, during the +// bottom diff layers are merging which tags these as stale, the read +// happens via a pre-created top snapshot layer which tries to access +// the state in these stale layers. Ensure this read can retrieve the +// right state back(block until the flattening is finished) instead of +// an unexpected error(snapshot layer is stale). +func TestReadStateDuringFlattening(t *testing.T) { + // setAccount is a helper to construct a random account entry and assign it to + // an account slot in a snapshot + setAccount := func(accKey string) map[common.Hash][]byte { + return map[common.Hash][]byte{ + common.HexToHash(accKey): randomAccount(), + } + } + // Create a starting base layer and a snapshot tree out of it + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + // 4 layers in total, 3 diff layers and 1 disk layers + snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), nil, setAccount("0xa1"), nil) + snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), nil, setAccount("0xa2"), nil) + snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil) + + // Obtain the topmost snapshot handler for state accessing + snap := snaps.Snapshot(common.HexToHash("0xa3")) + + // Register the testing hook to access the state after flattening + var result = make(chan *Account) + snaps.onFlatten = func() { + // Spin up a thread to read the account from the pre-created + // snapshot handler. It's expected to be blocked. + go func() { + account, _ := snap.Account(common.HexToHash("0xa1")) + result <- account + }() + select { + case res := <-result: + t.Fatalf("Unexpected return %v", res) + case <-time.NewTimer(time.Millisecond * 300).C: + } + } + // Cap the snap tree, which will mark the bottom-most layer as stale. + snaps.Cap(common.HexToHash("0xa3"), 1) + select { + case account := <-result: + if account == nil { + t.Fatal("Failed to retrieve account") + } + case <-time.NewTimer(time.Millisecond * 300).C: + t.Fatal("Unexpected blocker") + } +} diff --git a/core/state/snapshot/sort.go b/core/state/snapshot/sort.go new file mode 100644 index 0000000000..88841231d9 --- /dev/null +++ b/core/state/snapshot/sort.go @@ -0,0 +1,36 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "bytes" + + "github.com/ethereum/go-ethereum/common" +) + +// hashes is a helper to implement sort.Interface. +type hashes []common.Hash + +// Len is the number of elements in the collection. +func (hs hashes) Len() int { return len(hs) } + +// Less reports whether the element with index i should sort before the element +// with index j. +func (hs hashes) Less(i, j int) bool { return bytes.Compare(hs[i][:], hs[j][:]) < 0 } + +// Swap swaps the elements with indexes i and j. +func (hs hashes) Swap(i, j int) { hs[i], hs[j] = hs[j], hs[i] } diff --git a/core/state/snapshot/utils.go b/core/state/snapshot/utils.go new file mode 100644 index 0000000000..68c049a427 --- /dev/null +++ b/core/state/snapshot/utils.go @@ -0,0 +1,165 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "bytes" + "fmt" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" + "github.com/harmony-one/harmony/core/rawdb" + "github.com/harmony-one/harmony/internal/utils" +) + +// CheckDanglingStorage iterates the snap storage data, and verifies that all +// storage also has corresponding account data. +func CheckDanglingStorage(chaindb ethdb.KeyValueStore) error { + if err := checkDanglingDiskStorage(chaindb); err != nil { + utils.Logger().Error().Err(err).Msg("Database check error") + } + return checkDanglingMemStorage(chaindb) +} + +// checkDanglingDiskStorage checks if there is any 'dangling' storage data in the +// disk-backed snapshot layer. +func checkDanglingDiskStorage(chaindb ethdb.KeyValueStore) error { + var ( + lastReport = time.Now() + start = time.Now() + lastKey []byte + it = rawdb.NewKeyLengthIterator(chaindb.NewIterator(rawdb.SnapshotStoragePrefix, nil), 1+2*common.HashLength) + ) + utils.Logger().Info().Msg("Checking dangling snapshot disk storage") + + defer it.Release() + for it.Next() { + k := it.Key() + accKey := k[1:33] + if bytes.Equal(accKey, lastKey) { + // No need to look up for every slot + continue + } + lastKey = common.CopyBytes(accKey) + if time.Since(lastReport) > time.Second*8 { + utils.Logger().Info(). + Str("at", fmt.Sprintf("%#x", accKey)). + Interface("elapsed", common.PrettyDuration(time.Since(start))). + Msg("Iterating snap storage") + lastReport = time.Now() + } + if data := rawdb.ReadAccountSnapshot(chaindb, common.BytesToHash(accKey)); len(data) == 0 { + utils.Logger().Warn(). + Str("account", fmt.Sprintf("%#x", accKey)). + Str("storagekey", fmt.Sprintf("%#x", k)). + Msg("Dangling storage - missing account") + + return fmt.Errorf("dangling snapshot storage account %#x", accKey) + } + } + utils.Logger().Info().Err(it.Error()). + Interface("time", common.PrettyDuration(time.Since(start))). + Msg("Verified the snapshot disk storage") + + return nil +} + +// checkDanglingMemStorage checks if there is any 'dangling' storage in the journalled +// snapshot difflayers. +func checkDanglingMemStorage(db ethdb.KeyValueStore) error { + start := time.Now() + utils.Logger().Info().Msg("Checking dangling journalled storage") + err := iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + for accHash := range storage { + if _, ok := accounts[accHash]; !ok { + utils.Logger().Error(). + Str("account", fmt.Sprintf("%#x", accHash)). + Interface("root", root). + Msg("Dangling storage - missing account") + } + } + return nil + }) + if err != nil { + utils.Logger().Info().Err(err).Msg("Failed to resolve snapshot journal") + return err + } + utils.Logger().Info().Interface("time", common.PrettyDuration(time.Since(start))).Msg("Verified the snapshot journalled storage") + return nil +} + +// CheckJournalAccount shows information about an account, from the disk layer and +// up through the diff layers. +func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error { + // Look up the disk layer first + baseRoot := rawdb.ReadSnapshotRoot(db) + fmt.Printf("Disklayer: Root: %x\n", baseRoot) + if data := rawdb.ReadAccountSnapshot(db, hash); data != nil { + account := new(Account) + if err := rlp.DecodeBytes(data, account); err != nil { + panic(err) + } + fmt.Printf("\taccount.nonce: %d\n", account.Nonce) + fmt.Printf("\taccount.balance: %x\n", account.Balance) + fmt.Printf("\taccount.root: %x\n", account.Root) + fmt.Printf("\taccount.codehash: %x\n", account.CodeHash) + } + // Check storage + { + it := rawdb.NewKeyLengthIterator(db.NewIterator(append(rawdb.SnapshotStoragePrefix, hash.Bytes()...), nil), 1+2*common.HashLength) + fmt.Printf("\tStorage:\n") + for it.Next() { + slot := it.Key()[33:] + fmt.Printf("\t\t%x: %x\n", slot, it.Value()) + } + it.Release() + } + var depth = 0 + + return iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + _, a := accounts[hash] + _, b := destructs[hash] + _, c := storage[hash] + depth++ + if !a && !b && !c { + return nil + } + fmt.Printf("Disklayer+%d: Root: %x, parent %x\n", depth, root, pRoot) + if data, ok := accounts[hash]; ok { + account := new(Account) + if err := rlp.DecodeBytes(data, account); err != nil { + panic(err) + } + fmt.Printf("\taccount.nonce: %d\n", account.Nonce) + fmt.Printf("\taccount.balance: %x\n", account.Balance) + fmt.Printf("\taccount.root: %x\n", account.Root) + fmt.Printf("\taccount.codehash: %x\n", account.CodeHash) + } + if _, ok := destructs[hash]; ok { + fmt.Printf("\t Destructed!") + } + if data, ok := storage[hash]; ok { + fmt.Printf("\tStorage\n") + for k, v := range data { + fmt.Printf("\t\t%x: %x\n", k, v) + } + } + return nil + }) +} From d9bc7a87217de5edf221a13d59361b9ea1afdde3 Mon Sep 17 00:00:00 2001 From: Konstantin <355847+Frozen@users.noreply.github.com> Date: Mon, 24 Apr 2023 12:31:28 -0500 Subject: [PATCH 392/420] Typed cache & Node cleanup. (#4409) * Channels usage through methods. * Fix retry count. Removed proposedBlock. * keysToAddrs rewritten to lrucache. --- api/service/blockproposal/service.go | 12 ++++---- consensus/consensus.go | 24 ++++++++++++--- consensus/consensus_service.go | 4 +-- consensus/consensus_test.go | 4 +-- consensus/consensus_v2.go | 12 ++++---- consensus/view_change.go | 5 +--- core/evm.go | 4 --- internal/utils/lrucache/lrucache.go | 27 +++++++++++++++++ node/node.go | 44 ++++++++++------------------ node/node_newblock.go | 28 +++++------------- node/service_setup.go | 2 +- 11 files changed, 87 insertions(+), 79 deletions(-) create mode 100644 internal/utils/lrucache/lrucache.go diff --git a/api/service/blockproposal/service.go b/api/service/blockproposal/service.go index 1cbb5accf3..82db7c63a7 100644 --- a/api/service/blockproposal/service.go +++ b/api/service/blockproposal/service.go @@ -10,17 +10,15 @@ import ( type Service struct { stopChan chan struct{} stoppedChan chan struct{} - readySignal chan consensus.ProposalType - commitSigsChan chan []byte + c *consensus.Consensus messageChan chan *msg_pb.Message - waitForConsensusReady func(readySignal chan consensus.ProposalType, commitSigsChan chan []byte, stopChan chan struct{}, stoppedChan chan struct{}) + waitForConsensusReady func(c *consensus.Consensus, stopChan chan struct{}, stoppedChan chan struct{}) } // New returns a block proposal service. -func New(readySignal chan consensus.ProposalType, commitSigsChan chan []byte, waitForConsensusReady func(readySignal chan consensus.ProposalType, commitSigsChan chan []byte, stopChan chan struct{}, stoppedChan chan struct{})) *Service { +func New(c *consensus.Consensus, waitForConsensusReady func(c *consensus.Consensus, stopChan chan struct{}, stoppedChan chan struct{})) *Service { return &Service{ - readySignal: readySignal, - commitSigsChan: commitSigsChan, + c: c, waitForConsensusReady: waitForConsensusReady, stopChan: make(chan struct{}), stoppedChan: make(chan struct{}), @@ -34,7 +32,7 @@ func (s *Service) Start() error { } func (s *Service) run() { - s.waitForConsensusReady(s.readySignal, s.commitSigsChan, s.stopChan, s.stoppedChan) + s.waitForConsensusReady(s.c, s.stopChan, s.stoppedChan) } // Stop stops block proposal service. diff --git a/consensus/consensus.go b/consensus/consensus.go index d5130e22c9..c5573c972d 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -87,9 +87,9 @@ type Consensus struct { // ViewChange struct vc *viewChange // Signal channel for proposing a new block and start new consensus - ReadySignal chan ProposalType + readySignal chan ProposalType // Channel to send full commit signatures to finish new block proposal - CommitSigChannel chan []byte + commitSigChannel chan []byte // The post-consensus job func passed from Node object // Called when consensus on a new block is done PostConsensusJob func(*types.Block) error @@ -139,6 +139,22 @@ func (consensus *Consensus) Blockchain() core.BlockChain { return consensus.registry.GetBlockchain() } +func (consensus *Consensus) ReadySignal(p ProposalType) { + consensus.readySignal <- p +} + +func (consensus *Consensus) GetReadySignal() chan ProposalType { + return consensus.readySignal +} + +func (consensus *Consensus) CommitSigChannel() chan []byte { + return consensus.commitSigChannel +} + +func (consensus *Consensus) GetCommitSigChannel() chan []byte { + return consensus.commitSigChannel +} + // VerifyBlock is a function used to verify the block and keep trace of verified blocks. func (consensus *Consensus) verifyBlock(block *types.Block) error { if !consensus.FBFTLog.IsBlockVerified(block.Hash()) { @@ -274,8 +290,8 @@ func New( consensus.SetCurBlockViewID(0) consensus.ShardID = shard consensus.SlashChan = make(chan slash.Record) - consensus.ReadySignal = make(chan ProposalType) - consensus.CommitSigChannel = make(chan []byte) + consensus.readySignal = make(chan ProposalType) + consensus.commitSigChannel = make(chan []byte) // channel for receiving newly generated VDF consensus.RndChannel = make(chan [vdfAndSeedSize]byte) consensus.IgnoreViewIDCheck = abool.NewBool(false) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 3ca29812d9..7ca863733f 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -458,10 +458,10 @@ func (consensus *Consensus) updateConsensusInformation() Mode { if (oldLeader != nil && consensus.LeaderPubKey != nil && !consensus.LeaderPubKey.Object.IsEqual(oldLeader.Object)) && consensus.isLeader() { go func() { - consensus.getLogger().Info(). + consensus.GetLogger().Info(). Str("myKey", myPubKeys.SerializeToHexStr()). Msg("[UpdateConsensusInformation] I am the New Leader") - consensus.ReadySignal <- SyncProposal + consensus.ReadySignal(SyncProposal) }() } return Normal diff --git a/consensus/consensus_test.go b/consensus/consensus_test.go index 41fe5b127f..d2397a64aa 100644 --- a/consensus/consensus_test.go +++ b/consensus/consensus_test.go @@ -66,8 +66,8 @@ func TestConsensusInitialization(t *testing.T) { assert.IsType(t, make(chan slash.Record), consensus.SlashChan) assert.NotNil(t, consensus.SlashChan) - assert.IsType(t, make(chan ProposalType), consensus.ReadySignal) - assert.NotNil(t, consensus.ReadySignal) + assert.IsType(t, make(chan ProposalType), consensus.GetReadySignal()) + assert.NotNil(t, consensus.GetReadySignal()) assert.IsType(t, make(chan [vdfAndSeedSize]byte), consensus.RndChannel) assert.NotNil(t, consensus.RndChannel) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index b00e0d0d30..5dfc0f691b 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -254,13 +254,13 @@ func (consensus *Consensus) finalCommit() { // No pipelining go func() { consensus.getLogger().Info().Msg("[finalCommit] sending block proposal signal") - consensus.ReadySignal <- SyncProposal + consensus.ReadySignal(SyncProposal) }() } else { // pipelining go func() { select { - case consensus.CommitSigChannel <- commitSigAndBitmap: + case consensus.GetCommitSigChannel() <- commitSigAndBitmap: case <-time.After(CommitSigSenderTimeout): utils.Logger().Error().Err(err).Msg("[finalCommit] channel not received after 6s for commitSigAndBitmap") } @@ -334,7 +334,7 @@ func (consensus *Consensus) StartChannel() { consensus.start = true consensus.getLogger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Send ReadySignal") consensus.mutex.Unlock() - consensus.ReadySignal <- SyncProposal + consensus.ReadySignal(SyncProposal) return } consensus.mutex.Unlock() @@ -428,7 +428,7 @@ func (consensus *Consensus) BlockChannel(newBlock *types.Block) { return } // Sleep to wait for the full block time - consensus.getLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") + consensus.GetLogger().Info().Msg("[ConsensusMainLoop] Waiting for Block Time") time.AfterFunc(time.Until(consensus.NextBlockDue), func() { consensus.StartFinalityCount() consensus.mutex.Lock() @@ -587,7 +587,7 @@ func (consensus *Consensus) preCommitAndPropose(blk *types.Block) error { // Send signal to Node to propose the new block for consensus consensus.getLogger().Info().Msg("[preCommitAndPropose] sending block proposal signal") - consensus.ReadySignal <- AsyncProposal + consensus.ReadySignal(AsyncProposal) }() return nil @@ -761,7 +761,7 @@ func (consensus *Consensus) rotateLeader(epoch *big.Int) { if consensus.isLeader() && !consensus.getLeaderPubKey().Object.IsEqual(prev.Object) { // leader changed go func() { - consensus.ReadySignal <- SyncProposal + consensus.ReadySignal(SyncProposal) }() } } diff --git a/consensus/view_change.go b/consensus/view_change.go index 45838f9bc4..151074817a 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -422,10 +422,7 @@ func (consensus *Consensus) onViewChange(recvMsg *FBFTMessage) { consensus.getLogger().Error().Err(err).Msg("[onViewChange] startNewView failed") return } - - go func() { - consensus.ReadySignal <- SyncProposal - }() + go consensus.ReadySignal(SyncProposal) return } diff --git a/core/evm.go b/core/evm.go index f395ab042f..e11726a569 100644 --- a/core/evm.go +++ b/core/evm.go @@ -26,7 +26,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/block" - consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/internal/params" @@ -39,9 +38,6 @@ import ( // ChainContext supports retrieving headers and consensus parameters from the // current blockchain to be used during transaction processing. type ChainContext interface { - // Engine retrieves the chain's consensus engine. - Engine() consensus_engine.Engine - // GetHeader returns the hash corresponding to their hash. GetHeader(common.Hash, uint64) *block.Header diff --git a/internal/utils/lrucache/lrucache.go b/internal/utils/lrucache/lrucache.go new file mode 100644 index 0000000000..4859811b51 --- /dev/null +++ b/internal/utils/lrucache/lrucache.go @@ -0,0 +1,27 @@ +package lrucache + +import lru "github.com/hashicorp/golang-lru" + +type Cache[K comparable, V any] struct { + cache *lru.Cache +} + +func NewCache[K comparable, V any](size int) *Cache[K, V] { + c, _ := lru.New(size) + return &Cache[K, V]{ + cache: c, + } +} + +func (c *Cache[K, V]) Get(key K) (V, bool) { + v, ok := c.cache.Get(key) + if !ok { + var out V + return out, false + } + return v.(V), true +} + +func (c *Cache[K, V]) Set(key K, value V) { + c.cache.Add(key, value) +} diff --git a/node/node.go b/node/node.go index c7a01bd76e..710751776f 100644 --- a/node/node.go +++ b/node/node.go @@ -17,6 +17,7 @@ import ( "github.com/harmony-one/harmony/internal/shardchain/tikv_manage" "github.com/harmony-one/harmony/internal/tikv" "github.com/harmony-one/harmony/internal/tikv/redis_helper" + "github.com/harmony-one/harmony/internal/utils/lrucache" "github.com/ethereum/go-ethereum/rlp" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" @@ -132,8 +133,7 @@ type Node struct { chainConfig params.ChainConfig unixTimeAtNodeStart int64 // KeysToAddrs holds the addresses of bls keys run by the node - KeysToAddrs map[string]common.Address - keysToAddrsEpoch *big.Int + keysToAddrs *lrucache.Cache[uint64, map[string]common.Address] keysToAddrsMutex sync.Mutex // TransactionErrorSink contains error messages for any failed transaction, in memory only TransactionErrorSink *types.TransactionErrorSink @@ -141,7 +141,6 @@ type Node struct { BroadcastInvalidTx bool // InSync flag indicates the node is in-sync or not IsSynchronized *abool.AtomicBool - proposedBlock map[uint64]*types.Block deciderCache *lru.Cache committeeCache *lru.Cache @@ -1024,6 +1023,7 @@ func New( TransactionErrorSink: types.NewTransactionErrorSink(), crosslinks: crosslinks.New(), syncID: GenerateSyncID(), + keysToAddrs: lrucache.NewCache[uint64, map[string]common.Address](10), } if consensusObj == nil { panic("consensusObj is nil") @@ -1113,7 +1113,6 @@ func New( node.committeeCache, _ = lru.New(16) node.pendingCXReceipts = map[string]*types.CXReceiptsProof{} - node.proposedBlock = map[uint64]*types.Block{} node.Consensus.VerifiedNewBlock = make(chan *types.Block, 1) // the sequence number is the next block number to be added in consensus protocol, which is // always one more than current chain header block @@ -1322,10 +1321,6 @@ func (node *Node) ShutDown() { } func (node *Node) populateSelfAddresses(epoch *big.Int) { - // reset the self addresses - node.KeysToAddrs = map[string]common.Address{} - node.keysToAddrsEpoch = epoch - shardID := node.Consensus.ShardID shardState, err := node.Consensus.Blockchain().ReadShardState(epoch) if err != nil { @@ -1344,7 +1339,7 @@ func (node *Node) populateSelfAddresses(epoch *big.Int) { Msg("[PopulateSelfAddresses] failed to find shard committee") return } - + keysToAddrs := map[string]common.Address{} for _, blskey := range node.Consensus.GetPublicKeys() { blsStr := blskey.Bytes.Hex() shardkey := bls.FromLibBLSPublicKeyUnsafe(blskey.Object) @@ -1365,7 +1360,7 @@ func (node *Node) populateSelfAddresses(epoch *big.Int) { Msg("[PopulateSelfAddresses] could not find address") return } - node.KeysToAddrs[blsStr] = *addr + keysToAddrs[blsStr] = *addr utils.Logger().Debug(). Int64("epoch", epoch.Int64()). Uint32("shard-id", shardID). @@ -1373,34 +1368,27 @@ func (node *Node) populateSelfAddresses(epoch *big.Int) { Str("address", common2.MustAddressToBech32(*addr)). Msg("[PopulateSelfAddresses]") } + node.keysToAddrs.Set(epoch.Uint64(), keysToAddrs) } // GetAddressForBLSKey retrieves the ECDSA address associated with bls key for epoch func (node *Node) GetAddressForBLSKey(blskey *bls_core.PublicKey, epoch *big.Int) common.Address { - // populate if first time setting or new epoch - node.keysToAddrsMutex.Lock() - defer node.keysToAddrsMutex.Unlock() - if node.keysToAddrsEpoch == nil || epoch.Cmp(node.keysToAddrsEpoch) != 0 { - node.populateSelfAddresses(epoch) - } - blsStr := blskey.SerializeToHexStr() - addr, ok := node.KeysToAddrs[blsStr] - if !ok { - return common.Address{} - } - return addr + return node.GetAddresses(epoch)[blskey.SerializeToHexStr()] } // GetAddresses retrieves all ECDSA addresses of the bls keys for epoch func (node *Node) GetAddresses(epoch *big.Int) map[string]common.Address { - // populate if first time setting or new epoch + // populate if new epoch + if rs, ok := node.keysToAddrs.Get(epoch.Uint64()); ok { + return rs + } node.keysToAddrsMutex.Lock() - defer node.keysToAddrsMutex.Unlock() - if node.keysToAddrsEpoch == nil || epoch.Cmp(node.keysToAddrsEpoch) != 0 { - node.populateSelfAddresses(epoch) + node.populateSelfAddresses(epoch) + node.keysToAddrsMutex.Unlock() + if rs, ok := node.keysToAddrs.Get(epoch.Uint64()); ok { + return rs } - // self addresses map can never be nil - return node.KeysToAddrs + return make(map[string]common.Address) } // IsRunningBeaconChain returns whether the node is running on beacon chain. diff --git a/node/node_newblock.go b/node/node_newblock.go index 5050e4d6a7..27c0d843bd 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -7,7 +7,6 @@ import ( "time" "github.com/harmony-one/harmony/consensus" - "github.com/harmony-one/harmony/crypto/bls" staking "github.com/harmony-one/harmony/staking/types" @@ -27,7 +26,7 @@ const ( // WaitForConsensusReadyV2 listen for the readiness signal from consensus and generate new block for consensus. // only leader will receive the ready signal -func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalType, commitSigsChan chan []byte, stopChan chan struct{}, stoppedChan chan struct{}) { +func (node *Node) WaitForConsensusReadyV2(cs *consensus.Consensus, stopChan chan struct{}, stoppedChan chan struct{}) { go func() { // Setup stoppedChan defer close(stoppedChan) @@ -47,12 +46,11 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp utils.Logger().Warn(). Msg("Consensus new block proposal: STOPPED!") return - case proposalType := <-readySignal: - retryCount := 0 - for node.Consensus != nil && node.Consensus.IsLeader() { + case proposalType := <-cs.GetReadySignal(): + for retryCount := 0; retryCount < 3 && cs.IsLeader(); retryCount++ { time.Sleep(SleepPeriod) utils.Logger().Info(). - Uint64("blockNum", node.Blockchain().CurrentBlock().NumberU64()+1). + Uint64("blockNum", cs.Blockchain().CurrentBlock().NumberU64()+1). Bool("asyncProposal", proposalType == consensus.AsyncProposal). Msg("PROPOSING NEW BLOCK ------------------------------------------------") @@ -71,14 +69,14 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp } else { utils.Logger().Info().Msg("[ProposeNewBlock] Timeout waiting for commit sigs, reading directly from DB") } - sigs, err := node.Consensus.BlockCommitSigs(node.Blockchain().CurrentBlock().NumberU64()) + sigs, err := cs.BlockCommitSigs(cs.Blockchain().CurrentBlock().NumberU64()) if err != nil { utils.Logger().Error().Err(err).Msg("[ProposeNewBlock] Cannot get commit signatures from last block") } else { newCommitSigsChan <- sigs } - case commitSigs := <-commitSigsChan: + case commitSigs := <-cs.CommitSigChannel(): utils.Logger().Info().Msg("[ProposeNewBlock] received commit sigs asynchronously") if len(commitSigs) > bls.BLSSignatureSizeInBytes { newCommitSigsChan <- commitSigs @@ -86,12 +84,7 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp } }() newBlock, err := node.ProposeNewBlock(newCommitSigsChan) - if err == nil { - if blk, ok := node.proposedBlock[newBlock.NumberU64()]; ok { - utils.Logger().Info().Uint64("blockNum", newBlock.NumberU64()).Str("blockHash", blk.Hash().Hex()). - Msg("Block with the same number was already proposed, abort.") - } utils.Logger().Info(). Uint64("blockNum", newBlock.NumberU64()). Uint64("epoch", newBlock.Epoch().Uint64()). @@ -102,18 +95,11 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp Msg("=========Successfully Proposed New Block==========") // Send the new block to Consensus so it can be confirmed. - node.proposedBlock[newBlock.NumberU64()] = newBlock - delete(node.proposedBlock, newBlock.NumberU64()-10) - node.Consensus.BlockChannel(newBlock) + cs.BlockChannel(newBlock) break } else { - retryCount++ utils.Logger().Err(err).Int("retryCount", retryCount). Msg("!!!!!!!!!Failed Proposing New Block!!!!!!!!!") - if retryCount > 3 { - // break to avoid repeated failures - break - } continue } } diff --git a/node/service_setup.go b/node/service_setup.go index a2518110c1..94a34d4f1c 100644 --- a/node/service_setup.go +++ b/node/service_setup.go @@ -19,7 +19,7 @@ func (node *Node) RegisterValidatorServices() { // Register new block service. node.serviceManager.Register( service.BlockProposal, - blockproposal.New(node.Consensus.ReadySignal, node.Consensus.CommitSigChannel, node.WaitForConsensusReadyV2), + blockproposal.New(node.Consensus, node.WaitForConsensusReadyV2), ) } From a2bd56e105545854e208b3936cfa8fff4f7ac045 Mon Sep 17 00:00:00 2001 From: Max <82761650+MaxMustermann2@users.noreply.github.com> Date: Wed, 26 Apr 2023 12:42:15 +0000 Subject: [PATCH 393/420] core, internal/configs: HIP28-v2 fee collection (#4410) * core, internal/configs: HIP28-v2 fee collection Based on the Snapshot vote that has passed, collect 50% of the fee to a community maintained account and the remainder to an account used to pay for infrastructure costs. Note that these accounts need to be decided and set in the code at this moment, and the feature needs to be activated by setting the `FeeCollectEpoch` of the `ChainConfig` object. The implementation for devnet is a bit different than compared to others because the feature was activated on devnet with 100% collection to an account. I have handled this case separately in `devnet.go`. * test: add test for StateTransition.ApplyMessage The objective of this unit test is to check that the fees of a transaction are appropriately credited to the fee collectors that are set. This means, for a transaction of 21,000 gas limit and 100 gwei gas price, two equal fee collectors get 10,500 * 100 gwei each. In addition, to be clear that the refund mechanism (in case a transaction with extra gas comes in) works, the tested transaction has a 50,000 gas limit of which only 21,000 gas limit is actually consumed. * sharding/config: clarify local fee collector pk * sharding/config: set testnet fee collector same as devnet * test: add test for truncated fee distribution * sharding/config: set fee collector addresses * test: hardcode the expected fee collected * goimports * params/config: set testnet fee collect epoch Schedule testnet hard fork epoch to be 1296, which begins around the time 2023-04-28 07:14:20+00:00 * params/config: schedule devnee fee collection Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> --- cmd/harmony/main.go | 2 +- consensus/quorum/quorom_test.go | 2 +- core/state_transition.go | 27 +++- core/state_transition_test.go | 131 ++++++++++++++++++++ internal/configs/sharding/instance.go | 29 +++-- internal/configs/sharding/localnet.go | 20 +-- internal/configs/sharding/mainnet.go | 47 +++---- internal/configs/sharding/pangaea.go | 4 +- internal/configs/sharding/partner.go | 21 +++- internal/configs/sharding/shardingconfig.go | 5 +- internal/configs/sharding/stress.go | 6 +- internal/configs/sharding/testnet.go | 24 ++-- internal/params/config.go | 8 +- 13 files changed, 256 insertions(+), 70 deletions(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index f01cb758ed..88da5f7513 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -495,7 +495,7 @@ func nodeconfigSetShardSchedule(config harmonyconfig.HarmonyConfig) { } devnetConfig, err := shardingconfig.NewInstance( - uint32(dnConfig.NumShards), dnConfig.ShardSize, dnConfig.HmyNodeSize, dnConfig.SlotsLimit, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccounts, shardingconfig.Allowlist{}, ethCommon.Address{}, nil, shardingconfig.VLBPE) + uint32(dnConfig.NumShards), dnConfig.ShardSize, dnConfig.HmyNodeSize, dnConfig.SlotsLimit, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccounts, shardingconfig.Allowlist{}, nil, nil, shardingconfig.VLBPE) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "ERROR invalid devnet sharding config: %s", err) diff --git a/consensus/quorum/quorom_test.go b/consensus/quorum/quorom_test.go index b663b58d5b..09ec1779b8 100644 --- a/consensus/quorum/quorom_test.go +++ b/consensus/quorum/quorom_test.go @@ -565,7 +565,7 @@ func TestNthNextHmyExt(test *testing.T) { allLeaders := append(blsKeys[:numHmyNodes], allowlistLeaders...) decider := NewDecider(SuperMajorityVote, shard.BeaconChainShardID) - fakeInstance := shardingconfig.MustNewInstance(2, 20, numHmyNodes, 0, numeric.OneDec(), nil, nil, allowlist, common.Address{}, nil, 0) + fakeInstance := shardingconfig.MustNewInstance(2, 20, numHmyNodes, 0, numeric.OneDec(), nil, nil, allowlist, nil, nil, 0) decider.UpdateParticipants(blsKeys, allowlistLeaders) for i := 0; i < len(allLeaders); i++ { diff --git a/core/state_transition.go b/core/state_transition.go index c3ad10d496..9684812cbb 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -25,6 +25,7 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" stakingTypes "github.com/harmony-one/harmony/staking/types" "github.com/pkg/errors" @@ -281,14 +282,28 @@ func (st *StateTransition) refundGas() { } func (st *StateTransition) collectGas() { - // Burn Txn Fees after staking epoch if config := st.evm.ChainConfig(); !config.IsStaking(st.evm.EpochNumber) { - txFee := new(big.Int).Mul(new(big.Int).SetUint64(st.gasUsed()), st.gasPrice) + // Before staking epoch, add the fees to the block producer + txFee := new(big.Int).Mul( + new(big.Int).SetUint64(st.gasUsed()), + st.gasPrice, + ) st.state.AddBalance(st.evm.Coinbase, txFee) - } else if config.IsFeeCollectEpoch(st.evm.EpochNumber) { // collect Txn Fees to community-managed account. - txFee := new(big.Int).Mul(new(big.Int).SetUint64(st.gasUsed()), st.gasPrice) - txFeeCollector := shard.Schedule.InstanceForEpoch(st.evm.EpochNumber).FeeCollector() - st.state.AddBalance(txFeeCollector, txFee) + } else if feeCollectors := shard.Schedule.InstanceForEpoch( + st.evm.EpochNumber, + ).FeeCollectors(); len(feeCollectors) > 0 { + // The caller must ensure that the feeCollectors are accurately set + // at the appropriate epochs + txFee := numeric.NewDecFromBigInt( + new(big.Int).Mul( + new(big.Int).SetUint64(st.gasUsed()), + st.gasPrice, + ), + ) + for address, percent := range feeCollectors { + collectedFee := percent.Mul(txFee) + st.state.AddBalance(address, collectedFee.TruncateInt()) + } } } diff --git a/core/state_transition_test.go b/core/state_transition_test.go index 43b592c0d8..2a335ef779 100644 --- a/core/state_transition_test.go +++ b/core/state_transition_test.go @@ -4,11 +4,16 @@ import ( "crypto/ecdsa" "fmt" "math" + "math/big" "testing" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" + shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" "github.com/harmony-one/harmony/internal/params" + "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" "github.com/pkg/errors" ) @@ -81,3 +86,129 @@ func testApplyStakingMessage(test applyStakingMessageTest, t *testing.T) { } }) } + +func TestCollectGas(t *testing.T) { + key, _ := crypto.GenerateKey() + chain, db, header, _ := getTestEnvironment(*key) + header.SetEpoch(new(big.Int).Set(params.LocalnetChainConfig.FeeCollectEpoch)) + + // set the shard schedule so that fee collectors are available + shard.Schedule = shardingconfig.LocalnetSchedule + feeCollectors := shard.Schedule.InstanceForEpoch(header.Epoch()).FeeCollectors() + if len(feeCollectors) == 0 { + t.Fatal("No fee collectors set") + } + + tx := types.NewTransaction( + 0, // nonce + common.BytesToAddress([]byte("to")), + 0, // shardid + big.NewInt(1e18), // amount, 1 ONE + 50000, // gasLimit, intentionally higher than the 21000 required + big.NewInt(100e9), // gasPrice + []byte{}, // payload, intentionally empty + ) + from, _ := tx.SenderAddress() + initialBalance := big.NewInt(2e18) + db.AddBalance(from, initialBalance) + msg, _ := tx.AsMessage(types.NewEIP155Signer(common.Big2)) + ctx := NewEVMContext(msg, header, chain, nil /* coinbase is nil, no block reward */) + ctx.TxType = types.SameShardTx + + vmenv := vm.NewEVM(ctx, db, params.TestChainConfig, vm.Config{}) + gasPool := new(GasPool).AddGas(math.MaxUint64) + _, err := ApplyMessage(vmenv, msg, gasPool) + if err != nil { + t.Fatal(err) + } + + // check that sender got the exact expected balance + balance := db.GetBalance(from) + fee := new(big.Int).Mul(tx.GasPrice(), new(big.Int). + SetUint64( + 21000, // base fee for normal transfers + )) + feePlusAmount := new(big.Int).Add(fee, tx.Value()) + expectedBalance := new(big.Int).Sub(initialBalance, feePlusAmount) + if balance.Cmp(expectedBalance) != 0 { + t.Errorf("Balance mismatch for sender: got %v, expected %v", balance, expectedBalance) + } + + // check that the fee collectors got half of the fees each + expectedFeePerCollector := new(big.Int).Mul(tx.GasPrice(), + new(big.Int).SetUint64(21000/2)) + for collector := range feeCollectors { + balance := db.GetBalance(collector) + if balance.Cmp(expectedFeePerCollector) != 0 { + t.Errorf("Balance mismatch for collector %v: got %v, expected %v", + collector, balance, expectedFeePerCollector) + } + } + + // lastly, check the receiver's balance + balance = db.GetBalance(*tx.To()) + if balance.Cmp(tx.Value()) != 0 { + t.Errorf("Balance mismatch for receiver: got %v, expected %v", balance, tx.Value()) + } +} + +func TestCollectGasRounding(t *testing.T) { + // We want to test that the fee collectors get the correct amount of fees + // even if the total fee is not a multiple of the fee ratio. + // For example, if the fee ratio is 1:1, and the total fee is 1e9, then + // the fee collectors should get 0.5e9 each. + // If the gas is 1e9 + 1, then the fee collectors should get 0.5e9 each, + // with the extra 1 being dropped. This test checks for that. + // Such a situation can potentially occur, but is not an immediate concern + // since we require transactions to have a minimum gas price of 100 gwei + // which is always even (in wei) and can be divided across two collectors. + // Hypothetically, a gas price of 1 wei * gas used of (21,000 + odd number) + // could result in such a case which is well handled. + key, _ := crypto.GenerateKey() + chain, db, header, _ := getTestEnvironment(*key) + header.SetEpoch(new(big.Int).Set(params.LocalnetChainConfig.FeeCollectEpoch)) + + // set the shard schedule so that fee collectors are available + shard.Schedule = shardingconfig.LocalnetSchedule + feeCollectors := shard.Schedule.InstanceForEpoch(header.Epoch()).FeeCollectors() + if len(feeCollectors) == 0 { + t.Fatal("No fee collectors set") + } + + tx := types.NewTransaction( + 0, // nonce + common.BytesToAddress([]byte("to")), + 0, // shardid + big.NewInt(1e18), // amount, 1 ONE + 5, // gasLimit + big.NewInt(1), // gasPrice + []byte{}, // payload, intentionally empty + ) + from, _ := tx.SenderAddress() + initialBalance := big.NewInt(2e18) + db.AddBalance(from, initialBalance) + msg, _ := tx.AsMessage(types.NewEIP155Signer(common.Big2)) + ctx := NewEVMContext(msg, header, chain, nil /* coinbase is nil, no block reward */) + ctx.TxType = types.SameShardTx + + vmenv := vm.NewEVM(ctx, db, params.TestChainConfig, vm.Config{}) + gasPool := new(GasPool).AddGas(math.MaxUint64) + st := NewStateTransition(vmenv, msg, gasPool, nil) + // buy gas to set initial gas to 5: gasLimit * gasPrice + if err := st.buyGas(); err != nil { + t.Fatal(err) + } + // set left over gas to 0, so gasUsed is 5 + st.gas = 0 + st.collectGas() + + // check that the fee collectors got the fees in the provided ratio + expectedFeePerCollector := big.NewInt(2) // GIF(5 / 2) = 2 + for collector := range feeCollectors { + balance := db.GetBalance(collector) + if balance.Cmp(expectedFeePerCollector) != 0 { + t.Errorf("Balance mismatch for collector %v: got %v, expected %v", + collector, balance, expectedFeePerCollector) + } + } +} diff --git a/internal/configs/sharding/instance.go b/internal/configs/sharding/instance.go index cf5919a685..90076e595b 100644 --- a/internal/configs/sharding/instance.go +++ b/internal/configs/sharding/instance.go @@ -36,9 +36,11 @@ type instance struct { blocksPerEpoch uint64 slotsLimit int // HIP-16: The absolute number of maximum effective slots per shard limit for each validator. 0 means no limit. allowlist Allowlist - feeCollector ethCommon.Address + feeCollectors FeeCollectors } +type FeeCollectors map[ethCommon.Address]numeric.Dec + // NewInstance creates and validates a new sharding configuration based // upon given parameters. func NewInstance( @@ -46,7 +48,7 @@ func NewInstance( hmyAccounts []genesis.DeployAccount, fnAccounts []genesis.DeployAccount, allowlist Allowlist, - feeCollector ethCommon.Address, + feeCollectors FeeCollectors, reshardingEpoch []*big.Int, blocksE uint64, ) (Instance, error) { if numShards < 1 { @@ -81,6 +83,17 @@ func NewInstance( "total voting power of harmony nodes should be within [0, 1]", ) } + if len(feeCollectors) > 0 { + total := numeric.ZeroDec() // is a copy + for _, v := range feeCollectors { + total = total.Add(v) + } + if !total.Equal(numeric.OneDec()) { + return nil, errors.Errorf( + "total fee collection percentage should be 1, but got %v", total, + ) + } + } return instance{ numShards: numShards, @@ -94,7 +107,7 @@ func NewInstance( reshardingEpoch: reshardingEpoch, blocksPerEpoch: blocksE, slotsLimit: slotsLimit, - feeCollector: feeCollector, + feeCollectors: feeCollectors, }, nil } @@ -108,13 +121,13 @@ func MustNewInstance( hmyAccounts []genesis.DeployAccount, fnAccounts []genesis.DeployAccount, allowlist Allowlist, - feeCollector ethCommon.Address, + feeCollectors FeeCollectors, reshardingEpoch []*big.Int, blocksPerEpoch uint64, ) Instance { slotsLimit := int(float32(numNodesPerShard-numHarmonyOperatedNodesPerShard) * slotsLimitPercent) sc, err := NewInstance( numShards, numNodesPerShard, numHarmonyOperatedNodesPerShard, slotsLimit, harmonyVotePercent, - hmyAccounts, fnAccounts, allowlist, feeCollector, reshardingEpoch, blocksPerEpoch, + hmyAccounts, fnAccounts, allowlist, feeCollectors, reshardingEpoch, blocksPerEpoch, ) if err != nil { panic(err) @@ -137,9 +150,9 @@ func (sc instance) SlotsLimit() int { return sc.slotsLimit } -// FeeCollector returns a address to receive txn fees -func (sc instance) FeeCollector() ethCommon.Address { - return sc.feeCollector +// FeeCollector returns a mapping of address to decimal % of fee +func (sc instance) FeeCollectors() FeeCollectors { + return sc.feeCollectors } // HarmonyVotePercent returns total percentage of voting power harmony nodes possess. diff --git a/internal/configs/sharding/localnet.go b/internal/configs/sharding/localnet.go index a31c66f0da..00ea1a7ac2 100644 --- a/internal/configs/sharding/localnet.go +++ b/internal/configs/sharding/localnet.go @@ -14,8 +14,12 @@ import ( // configuration schedule. var LocalnetSchedule localnetSchedule -// privatekey: 0x1111111111111111111111111111111111111111111111111111111111111111 -var feeCollectorLocalnet = mustAddress("0x19E7E376E7C213B7E7e7e46cc70A5dD086DAff2A") +var feeCollectorsLocalnet = FeeCollectors{ + // pk: 0x1111111111111111111111111111111111111111111111111111111111111111 + mustAddress("0x19E7E376E7C213B7E7e7e46cc70A5dD086DAff2A"): numeric.MustNewDecFromStr("0.5"), + // pk: 0x2222222222222222222222222222222222222222222222222222222222222222 + mustAddress("0x1563915e194D8CfBA1943570603F7606A3115508"): numeric.MustNewDecFromStr("0.5"), +} type localnetSchedule struct{} @@ -152,10 +156,10 @@ var ( big.NewInt(0), big.NewInt(localnetV1Epoch), params.LocalnetChainConfig.StakingEpoch, params.LocalnetChainConfig.TwoSecondsEpoch, } // Number of shards, how many slots on each , how many slots owned by Harmony - localnetV0 = MustNewInstance(2, 7, 5, 0, numeric.OneDec(), genesis.LocalHarmonyAccounts, genesis.LocalFnAccounts, emptyAllowlist, emptyAddress, localnetReshardingEpoch, LocalnetSchedule.BlocksPerEpochOld()) - localnetV1 = MustNewInstance(2, 8, 5, 0, numeric.OneDec(), genesis.LocalHarmonyAccountsV1, genesis.LocalFnAccountsV1, emptyAllowlist, emptyAddress, localnetReshardingEpoch, LocalnetSchedule.BlocksPerEpochOld()) - localnetV2 = MustNewInstance(2, 9, 6, 0, numeric.MustNewDecFromStr("0.68"), genesis.LocalHarmonyAccountsV2, genesis.LocalFnAccountsV2, emptyAllowlist, emptyAddress, localnetReshardingEpoch, LocalnetSchedule.BlocksPerEpochOld()) - localnetV3 = MustNewInstance(2, 9, 6, 0, numeric.MustNewDecFromStr("0.68"), genesis.LocalHarmonyAccountsV2, genesis.LocalFnAccountsV2, emptyAllowlist, emptyAddress, localnetReshardingEpoch, LocalnetSchedule.BlocksPerEpoch()) - localnetV3_1 = MustNewInstance(2, 9, 6, 0, numeric.MustNewDecFromStr("0.68"), genesis.LocalHarmonyAccountsV2, genesis.LocalFnAccountsV2, emptyAllowlist, emptyAddress, localnetReshardingEpoch, LocalnetSchedule.BlocksPerEpoch()) - localnetV3_2 = MustNewInstance(2, 9, 6, 0, numeric.MustNewDecFromStr("0.68"), genesis.LocalHarmonyAccountsV2, genesis.LocalFnAccountsV2, emptyAllowlist, feeCollectorLocalnet, localnetReshardingEpoch, LocalnetSchedule.BlocksPerEpoch()) + localnetV0 = MustNewInstance(2, 7, 5, 0, numeric.OneDec(), genesis.LocalHarmonyAccounts, genesis.LocalFnAccounts, emptyAllowlist, nil, localnetReshardingEpoch, LocalnetSchedule.BlocksPerEpochOld()) + localnetV1 = MustNewInstance(2, 8, 5, 0, numeric.OneDec(), genesis.LocalHarmonyAccountsV1, genesis.LocalFnAccountsV1, emptyAllowlist, nil, localnetReshardingEpoch, LocalnetSchedule.BlocksPerEpochOld()) + localnetV2 = MustNewInstance(2, 9, 6, 0, numeric.MustNewDecFromStr("0.68"), genesis.LocalHarmonyAccountsV2, genesis.LocalFnAccountsV2, emptyAllowlist, nil, localnetReshardingEpoch, LocalnetSchedule.BlocksPerEpochOld()) + localnetV3 = MustNewInstance(2, 9, 6, 0, numeric.MustNewDecFromStr("0.68"), genesis.LocalHarmonyAccountsV2, genesis.LocalFnAccountsV2, emptyAllowlist, nil, localnetReshardingEpoch, LocalnetSchedule.BlocksPerEpoch()) + localnetV3_1 = MustNewInstance(2, 9, 6, 0, numeric.MustNewDecFromStr("0.68"), genesis.LocalHarmonyAccountsV2, genesis.LocalFnAccountsV2, emptyAllowlist, nil, localnetReshardingEpoch, LocalnetSchedule.BlocksPerEpoch()) + localnetV3_2 = MustNewInstance(2, 9, 6, 0, numeric.MustNewDecFromStr("0.68"), genesis.LocalHarmonyAccountsV2, genesis.LocalFnAccountsV2, emptyAllowlist, feeCollectorsLocalnet, localnetReshardingEpoch, LocalnetSchedule.BlocksPerEpoch()) ) diff --git a/internal/configs/sharding/mainnet.go b/internal/configs/sharding/mainnet.go index 9e73a6f12b..fb6b0f13bc 100644 --- a/internal/configs/sharding/mainnet.go +++ b/internal/configs/sharding/mainnet.go @@ -47,9 +47,12 @@ var ( 3: []*big.Int{big.NewInt(183), big.NewInt(184), big.NewInt(185)}, } - emptyAddress = ethCommon.Address{} - // TODO: set a valid address - feeCollector ethCommon.Address // = mustAddress("0xXXX or onexxx") + feeCollectorsMainnet = FeeCollectors{ + // Infrastructure + mustAddress("0xa0c395A83503ad89613E43397e9fE1f8E93B6384"): numeric.MustNewDecFromStr("0.5"), + // Community + mustAddress("0xbdFeE8587d347Cd8df002E6154763325265Fa84c"): numeric.MustNewDecFromStr("0.5"), + } ) func mustAddress(addrStr string) ethCommon.Address { @@ -220,23 +223,23 @@ func (ms mainnetSchedule) IsSkippedEpoch(shardID uint32, epoch *big.Int) bool { var mainnetReshardingEpoch = []*big.Int{big.NewInt(0), big.NewInt(mainnetV0_1Epoch), big.NewInt(mainnetV0_2Epoch), big.NewInt(mainnetV0_3Epoch), big.NewInt(mainnetV0_4Epoch), big.NewInt(mainnetV1Epoch), big.NewInt(mainnetV1_1Epoch), big.NewInt(mainnetV1_2Epoch), big.NewInt(mainnetV1_3Epoch), big.NewInt(mainnetV1_4Epoch), big.NewInt(mainnetV1_5Epoch), big.NewInt(mainnetV2_0Epoch), big.NewInt(mainnetV2_1Epoch), big.NewInt(mainnetV2_2Epoch), params.MainnetChainConfig.TwoSecondsEpoch, params.MainnetChainConfig.SixtyPercentEpoch, params.MainnetChainConfig.HIP6And8Epoch} var ( - mainnetV0 = MustNewInstance(4, 150, 112, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccounts, emptyAllowlist, emptyAddress, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) - mainnetV0_1 = MustNewInstance(4, 152, 112, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV0_1, emptyAllowlist, emptyAddress, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) - mainnetV0_2 = MustNewInstance(4, 200, 148, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV0_2, emptyAllowlist, emptyAddress, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) - mainnetV0_3 = MustNewInstance(4, 210, 148, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV0_3, emptyAllowlist, emptyAddress, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) - mainnetV0_4 = MustNewInstance(4, 216, 148, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV0_4, emptyAllowlist, emptyAddress, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) - mainnetV1 = MustNewInstance(4, 250, 170, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1, emptyAllowlist, emptyAddress, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) - mainnetV1_1 = MustNewInstance(4, 250, 170, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_1, emptyAllowlist, emptyAddress, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) - mainnetV1_2 = MustNewInstance(4, 250, 170, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_2, emptyAllowlist, emptyAddress, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) - mainnetV1_3 = MustNewInstance(4, 250, 170, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_3, emptyAllowlist, emptyAddress, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) - mainnetV1_4 = MustNewInstance(4, 250, 170, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_4, emptyAllowlist, emptyAddress, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) - mainnetV1_5 = MustNewInstance(4, 250, 170, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_5, emptyAllowlist, emptyAddress, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) - mainnetV2_0 = MustNewInstance(4, 250, 170, 0, numeric.MustNewDecFromStr("0.68"), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_5, emptyAllowlist, emptyAddress, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) - mainnetV2_1 = MustNewInstance(4, 250, 130, 0, numeric.MustNewDecFromStr("0.68"), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_5, emptyAllowlist, emptyAddress, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) - mainnetV2_2 = MustNewInstance(4, 250, 90, 0, numeric.MustNewDecFromStr("0.68"), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_5, emptyAllowlist, emptyAddress, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) - mainnetV3 = MustNewInstance(4, 250, 90, 0, numeric.MustNewDecFromStr("0.68"), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_5, emptyAllowlist, emptyAddress, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpoch()) - mainnetV3_1 = MustNewInstance(4, 250, 50, 0, numeric.MustNewDecFromStr("0.60"), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_5, emptyAllowlist, emptyAddress, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpoch()) - mainnetV3_2 = MustNewInstance(4, 250, 25, 0, numeric.MustNewDecFromStr("0.49"), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_5, emptyAllowlist, emptyAddress, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpoch()) - mainnetV3_3 = MustNewInstance(4, 250, 25, 0.06, numeric.MustNewDecFromStr("0.49"), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_5, emptyAllowlist, emptyAddress, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpoch()) - mainnetV3_4 = MustNewInstance(4, 250, 25, 0.06, numeric.MustNewDecFromStr("0.49"), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_5, emptyAllowlist, feeCollector, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpoch()) + mainnetV0 = MustNewInstance(4, 150, 112, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccounts, emptyAllowlist, nil, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) + mainnetV0_1 = MustNewInstance(4, 152, 112, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV0_1, emptyAllowlist, nil, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) + mainnetV0_2 = MustNewInstance(4, 200, 148, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV0_2, emptyAllowlist, nil, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) + mainnetV0_3 = MustNewInstance(4, 210, 148, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV0_3, emptyAllowlist, nil, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) + mainnetV0_4 = MustNewInstance(4, 216, 148, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV0_4, emptyAllowlist, nil, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) + mainnetV1 = MustNewInstance(4, 250, 170, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1, emptyAllowlist, nil, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) + mainnetV1_1 = MustNewInstance(4, 250, 170, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_1, emptyAllowlist, nil, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) + mainnetV1_2 = MustNewInstance(4, 250, 170, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_2, emptyAllowlist, nil, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) + mainnetV1_3 = MustNewInstance(4, 250, 170, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_3, emptyAllowlist, nil, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) + mainnetV1_4 = MustNewInstance(4, 250, 170, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_4, emptyAllowlist, nil, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) + mainnetV1_5 = MustNewInstance(4, 250, 170, 0, numeric.OneDec(), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_5, emptyAllowlist, nil, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) + mainnetV2_0 = MustNewInstance(4, 250, 170, 0, numeric.MustNewDecFromStr("0.68"), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_5, emptyAllowlist, nil, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) + mainnetV2_1 = MustNewInstance(4, 250, 130, 0, numeric.MustNewDecFromStr("0.68"), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_5, emptyAllowlist, nil, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) + mainnetV2_2 = MustNewInstance(4, 250, 90, 0, numeric.MustNewDecFromStr("0.68"), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_5, emptyAllowlist, nil, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpochOld()) + mainnetV3 = MustNewInstance(4, 250, 90, 0, numeric.MustNewDecFromStr("0.68"), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_5, emptyAllowlist, nil, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpoch()) + mainnetV3_1 = MustNewInstance(4, 250, 50, 0, numeric.MustNewDecFromStr("0.60"), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_5, emptyAllowlist, nil, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpoch()) + mainnetV3_2 = MustNewInstance(4, 250, 25, 0, numeric.MustNewDecFromStr("0.49"), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_5, emptyAllowlist, nil, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpoch()) + mainnetV3_3 = MustNewInstance(4, 250, 25, 0.06, numeric.MustNewDecFromStr("0.49"), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_5, emptyAllowlist, nil, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpoch()) + mainnetV3_4 = MustNewInstance(4, 250, 25, 0.06, numeric.MustNewDecFromStr("0.49"), genesis.HarmonyAccounts, genesis.FoundationalNodeAccountsV1_5, emptyAllowlist, feeCollectorsMainnet, mainnetReshardingEpoch, MainnetSchedule.BlocksPerEpoch()) ) diff --git a/internal/configs/sharding/pangaea.go b/internal/configs/sharding/pangaea.go index a5542296c5..12ffc7fe59 100644 --- a/internal/configs/sharding/pangaea.go +++ b/internal/configs/sharding/pangaea.go @@ -75,5 +75,5 @@ var pangaeaReshardingEpoch = []*big.Int{ params.PangaeaChainConfig.StakingEpoch, } -var pangaeaV0 = MustNewInstance(4, 30, 30, 0, numeric.OneDec(), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, emptyAddress, pangaeaReshardingEpoch, PangaeaSchedule.BlocksPerEpoch()) -var pangaeaV1 = MustNewInstance(4, 110, 30, 0, numeric.MustNewDecFromStr("0.68"), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, emptyAddress, pangaeaReshardingEpoch, PangaeaSchedule.BlocksPerEpoch()) +var pangaeaV0 = MustNewInstance(4, 30, 30, 0, numeric.OneDec(), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, nil, pangaeaReshardingEpoch, PangaeaSchedule.BlocksPerEpoch()) +var pangaeaV1 = MustNewInstance(4, 110, 30, 0, numeric.MustNewDecFromStr("0.68"), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, nil, pangaeaReshardingEpoch, PangaeaSchedule.BlocksPerEpoch()) diff --git a/internal/configs/sharding/partner.go b/internal/configs/sharding/partner.go index 96dd584953..9cd6f69305 100644 --- a/internal/configs/sharding/partner.go +++ b/internal/configs/sharding/partner.go @@ -13,7 +13,17 @@ import ( // configuration schedule. var PartnerSchedule partnerSchedule -var feeCollectorDevnet = mustAddress("0xb728AEaBF60fD01816ee9e756c18bc01dC91ba5D") +var feeCollectEpochV1 = big.NewInt(574) + +var feeCollectorsDevnet = []FeeCollectors{ + FeeCollectors{ + mustAddress("0xb728AEaBF60fD01816ee9e756c18bc01dC91ba5D"): numeric.OneDec(), + }, + FeeCollectors{ + mustAddress("0xb728AEaBF60fD01816ee9e756c18bc01dC91ba5D"): numeric.MustNewDecFromStr("0.5"), + mustAddress("0xb41B6B8d9e68fD44caC8342BC2EEf4D59531d7d7"): numeric.MustNewDecFromStr("0.5"), + }, +} type partnerSchedule struct{} @@ -32,6 +42,8 @@ const ( func (ps partnerSchedule) InstanceForEpoch(epoch *big.Int) Instance { switch { case params.PartnerChainConfig.IsFeeCollectEpoch(epoch): + return partnerV3 + case epoch.Cmp(feeCollectEpochV1) >= 0: return partnerV2 case epoch.Cmp(params.PartnerChainConfig.StakingEpoch) >= 0: return partnerV1 @@ -80,6 +92,7 @@ var partnerReshardingEpoch = []*big.Int{ params.PartnerChainConfig.StakingEpoch, } -var partnerV0 = MustNewInstance(2, 5, 5, 0, numeric.OneDec(), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, emptyAddress, partnerReshardingEpoch, PartnerSchedule.BlocksPerEpoch()) -var partnerV1 = MustNewInstance(2, 5, 4, 0, numeric.MustNewDecFromStr("0.9"), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, emptyAddress, partnerReshardingEpoch, PartnerSchedule.BlocksPerEpoch()) -var partnerV2 = MustNewInstance(2, 5, 4, 0, numeric.MustNewDecFromStr("0.9"), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, feeCollectorDevnet, partnerReshardingEpoch, PartnerSchedule.BlocksPerEpoch()) +var partnerV0 = MustNewInstance(2, 5, 5, 0, numeric.OneDec(), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, nil, partnerReshardingEpoch, PartnerSchedule.BlocksPerEpoch()) +var partnerV1 = MustNewInstance(2, 5, 4, 0, numeric.MustNewDecFromStr("0.9"), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, nil, partnerReshardingEpoch, PartnerSchedule.BlocksPerEpoch()) +var partnerV2 = MustNewInstance(2, 5, 4, 0, numeric.MustNewDecFromStr("0.9"), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, feeCollectorsDevnet[0], partnerReshardingEpoch, PartnerSchedule.BlocksPerEpoch()) +var partnerV3 = MustNewInstance(2, 5, 4, 0, numeric.MustNewDecFromStr("0.9"), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, feeCollectorsDevnet[1], partnerReshardingEpoch, PartnerSchedule.BlocksPerEpoch()) diff --git a/internal/configs/sharding/shardingconfig.go b/internal/configs/sharding/shardingconfig.go index 91b4458dd3..577839dfc1 100644 --- a/internal/configs/sharding/shardingconfig.go +++ b/internal/configs/sharding/shardingconfig.go @@ -9,7 +9,6 @@ import ( "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/numeric" - ethCommon "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/harmony/internal/genesis" ) @@ -84,8 +83,8 @@ type Instance interface { // ExternalAllowlistLimit returns the maximum number of external leader keys on each shard(HIP18) ExternalAllowlistLimit() int - // FeeCollector returns a address to receive txn fees - FeeCollector() ethCommon.Address + // FeeCollector returns a mapping of address to decimal % of fee + FeeCollectors() FeeCollectors } // genShardingStructure return sharding structure, given shard number and its patterns. diff --git a/internal/configs/sharding/stress.go b/internal/configs/sharding/stress.go index e4e511fd65..70c294c637 100644 --- a/internal/configs/sharding/stress.go +++ b/internal/configs/sharding/stress.go @@ -78,6 +78,6 @@ var stressnetReshardingEpoch = []*big.Int{ params.StressnetChainConfig.StakingEpoch, } -var stressnetV0 = MustNewInstance(2, 10, 10, 0, numeric.OneDec(), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, emptyAddress, stressnetReshardingEpoch, StressNetSchedule.BlocksPerEpoch()) -var stressnetV1 = MustNewInstance(2, 30, 10, 0, numeric.MustNewDecFromStr("0.9"), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, emptyAddress, stressnetReshardingEpoch, StressNetSchedule.BlocksPerEpoch()) -var stressnetV2 = MustNewInstance(2, 30, 10, 0, numeric.MustNewDecFromStr("0.6"), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, emptyAddress, stressnetReshardingEpoch, StressNetSchedule.BlocksPerEpoch()) +var stressnetV0 = MustNewInstance(2, 10, 10, 0, numeric.OneDec(), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, nil, stressnetReshardingEpoch, StressNetSchedule.BlocksPerEpoch()) +var stressnetV1 = MustNewInstance(2, 30, 10, 0, numeric.MustNewDecFromStr("0.9"), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, nil, stressnetReshardingEpoch, StressNetSchedule.BlocksPerEpoch()) +var stressnetV2 = MustNewInstance(2, 30, 10, 0, numeric.MustNewDecFromStr("0.6"), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, nil, stressnetReshardingEpoch, StressNetSchedule.BlocksPerEpoch()) diff --git a/internal/configs/sharding/testnet.go b/internal/configs/sharding/testnet.go index 5892748c84..6ef55775e9 100644 --- a/internal/configs/sharding/testnet.go +++ b/internal/configs/sharding/testnet.go @@ -12,8 +12,13 @@ import ( // configuration schedule. var TestnetSchedule testnetSchedule -var TestnetNinetyPercentEpoch = big.NewInt(399) -var ShardReductionEpoch = big.NewInt(486) +var ninetyPercentEpoch = big.NewInt(399) +var shardReductionEpoch = big.NewInt(486) + +var feeCollectorsTestnet = FeeCollectors{ + mustAddress("0xb728AEaBF60fD01816ee9e756c18bc01dC91ba5D"): numeric.MustNewDecFromStr("0.5"), + mustAddress("0xb41B6B8d9e68fD44caC8342BC2EEf4D59531d7d7"): numeric.MustNewDecFromStr("0.5"), +} type testnetSchedule struct{} @@ -34,9 +39,11 @@ const ( func (ts testnetSchedule) InstanceForEpoch(epoch *big.Int) Instance { switch { - case epoch.Cmp(ShardReductionEpoch) >= 0: + case params.TestnetChainConfig.IsFeeCollectEpoch(epoch): + return testnetV4 + case epoch.Cmp(shardReductionEpoch) >= 0: return testnetV3 - case epoch.Cmp(TestnetNinetyPercentEpoch) >= 0: + case epoch.Cmp(ninetyPercentEpoch) >= 0: return testnetV2 case params.TestnetChainConfig.IsStaking(epoch): return testnetV1 @@ -114,8 +121,9 @@ var testnetReshardingEpoch = []*big.Int{ } var ( - testnetV0 = MustNewInstance(4, 8, 8, 0, numeric.OneDec(), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, emptyAddress, testnetReshardingEpoch, TestnetSchedule.BlocksPerEpoch()) - testnetV1 = MustNewInstance(4, 30, 8, 0.15, numeric.MustNewDecFromStr("0.70"), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, emptyAddress, testnetReshardingEpoch, TestnetSchedule.BlocksPerEpoch()) - testnetV2 = MustNewInstance(4, 30, 8, 0.15, numeric.MustNewDecFromStr("0.90"), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, emptyAddress, testnetReshardingEpoch, TestnetSchedule.BlocksPerEpoch()) - testnetV3 = MustNewInstance(2, 30, 8, 0.15, numeric.MustNewDecFromStr("0.90"), genesis.TNHarmonyAccountsV1, genesis.TNFoundationalAccounts, emptyAllowlist, emptyAddress, testnetReshardingEpoch, TestnetSchedule.BlocksPerEpoch()) + testnetV0 = MustNewInstance(4, 8, 8, 0, numeric.OneDec(), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, nil, testnetReshardingEpoch, TestnetSchedule.BlocksPerEpoch()) + testnetV1 = MustNewInstance(4, 30, 8, 0.15, numeric.MustNewDecFromStr("0.70"), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, nil, testnetReshardingEpoch, TestnetSchedule.BlocksPerEpoch()) + testnetV2 = MustNewInstance(4, 30, 8, 0.15, numeric.MustNewDecFromStr("0.90"), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, nil, testnetReshardingEpoch, TestnetSchedule.BlocksPerEpoch()) + testnetV3 = MustNewInstance(2, 30, 8, 0.15, numeric.MustNewDecFromStr("0.90"), genesis.TNHarmonyAccountsV1, genesis.TNFoundationalAccounts, emptyAllowlist, nil, testnetReshardingEpoch, TestnetSchedule.BlocksPerEpoch()) + testnetV4 = MustNewInstance(2, 30, 8, 0.15, numeric.MustNewDecFromStr("0.90"), genesis.TNHarmonyAccountsV1, genesis.TNFoundationalAccounts, emptyAllowlist, feeCollectorsTestnet, testnetReshardingEpoch, TestnetSchedule.BlocksPerEpoch()) ) diff --git a/internal/params/config.go b/internal/params/config.go index a34dcd3973..cc2f680d74 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -112,7 +112,7 @@ var ( AllowlistEpoch: big.NewInt(2), LeaderRotationExternalNonBeaconLeaders: EpochTBD, LeaderRotationExternalBeaconLeaders: EpochTBD, - FeeCollectEpoch: EpochTBD, + FeeCollectEpoch: big.NewInt(1296), // 2023-04-28 07:14:20+00:00 ValidatorCodeFixEpoch: EpochTBD, } // PangaeaChainConfig contains the chain parameters for the Pangaea network. @@ -193,10 +193,10 @@ var ( SlotsLimitedEpoch: EpochTBD, // epoch to enable HIP-16 CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, - FeeCollectEpoch: big.NewInt(574), + FeeCollectEpoch: big.NewInt(848), // 2023-04-28 04:33:33+00:00 LeaderRotationExternalNonBeaconLeaders: EpochTBD, LeaderRotationExternalBeaconLeaders: EpochTBD, - ValidatorCodeFixEpoch: EpochTBD, + ValidatorCodeFixEpoch: big.NewInt(848), } // StressnetChainConfig contains the chain parameters for the Stress test network. @@ -278,7 +278,7 @@ var ( AllowlistEpoch: EpochTBD, LeaderRotationExternalNonBeaconLeaders: big.NewInt(5), LeaderRotationExternalBeaconLeaders: big.NewInt(6), - FeeCollectEpoch: big.NewInt(5), + FeeCollectEpoch: big.NewInt(2), ValidatorCodeFixEpoch: EpochTBD, } From 550a022ec148d11942cac25b3c0cc6820722e429 Mon Sep 17 00:00:00 2001 From: Konstantin <355847+Frozen@users.noreply.github.com> Date: Wed, 26 Apr 2023 11:03:01 -0500 Subject: [PATCH 394/420] Minor: removed time.Sleep from tests. (#4412) * Provide current time as argument. * Fix test. * Fix naming. --- consensus/consensus_v2.go | 2 +- internal/utils/timer.go | 6 +++--- internal/utils/timer_test.go | 26 ++++++++++++++++++-------- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 5dfc0f691b..8e72896c3e 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -402,7 +402,7 @@ func (consensus *Consensus) tick() { continue } } - if !v.CheckExpire() { + if !v.Expired(time.Now()) { continue } if k != timeoutViewChange { diff --git a/internal/utils/timer.go b/internal/utils/timer.go index 2e8a77667b..d355d5c719 100644 --- a/internal/utils/timer.go +++ b/internal/utils/timer.go @@ -39,9 +39,9 @@ func (timeout *Timeout) Stop() { timeout.start = time.Now() } -// CheckExpire checks whether the timeout is reached/expired -func (timeout *Timeout) CheckExpire() bool { - if timeout.state == Active && time.Since(timeout.start) > timeout.d { +// Expired checks whether the timeout is reached/expired +func (timeout *Timeout) Expired(now time.Time) bool { + if timeout.state == Active && now.Sub(timeout.start) > timeout.d { timeout.state = Expired } if timeout.state == Expired { diff --git a/internal/utils/timer_test.go b/internal/utils/timer_test.go index 8b3061ddb6..cdd0dbd37f 100644 --- a/internal/utils/timer_test.go +++ b/internal/utils/timer_test.go @@ -15,17 +15,27 @@ func TestNewTimeout(t *testing.T) { func TestCheckExpire(t *testing.T) { timer := NewTimeout(time.Second) timer.Start() - time.Sleep(2 * time.Second) - if timer.CheckExpire() == false { - t.Fatalf("CheckExpire should be true") + now := time.Now() + if timer.Expired(now) { + t.Fatalf("Timer shouldn't be expired") } + if !timer.Expired(now.Add(2 * time.Second)) { + t.Fatalf("Timer should be expired") + } + // start again timer.Start() - if timer.CheckExpire() == true { - t.Fatalf("CheckExpire should be false") + if timer.Expired(now) { + t.Fatalf("Timer shouldn't be expired") + } + if !timer.Expired(now.Add(2 * time.Second)) { + t.Fatalf("Timer should be expired") } + // stop timer.Stop() - if timer.CheckExpire() == true { - t.Fatalf("CheckExpire should be false") + if timer.Expired(now) { + t.Fatalf("Timer shouldn't be expired because it is stopped") + } + if timer.Expired(now.Add(2 * time.Second)) { + t.Fatalf("Timer shouldn't be expired because it is stopped") } - } From cb199d97064dddb9f38dd62c0f6de5a1e63f35d4 Mon Sep 17 00:00:00 2001 From: Max <82761650+MaxMustermann2@users.noreply.github.com> Date: Wed, 26 Apr 2023 16:40:07 +0000 Subject: [PATCH 395/420] Mainnet Release Candidate 2023.1.2 (#4376) (#4414) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * Bump github.com/aws/aws-sdk-go from 1.33.0 to 1.34.0 Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.33.0 to 1.34.0. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/v1.34.0/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.33.0...v1.34.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production ... * Bump github.com/ipld/go-ipld-prime from 0.9.0 to 0.19.0 Bumps [github.com/ipld/go-ipld-prime](https://github.com/ipld/go-ipld-prime) from 0.9.0 to 0.19.0. - [Release notes](https://github.com/ipld/go-ipld-prime/releases) - [Changelog](https://github.com/ipld/go-ipld-prime/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipld/go-ipld-prime/compare/v0.9.0...v0.19.0) --- updated-dependencies: - dependency-name: github.com/ipld/go-ipld-prime dependency-type: indirect ... * Bump golang.org/x/net from 0.3.0 to 0.7.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.3.0 to 0.7.0. - [Release notes](https://github.com/golang/net/releases) - [Commits](https://github.com/golang/net/compare/v0.3.0...v0.7.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... * Small fixes. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * activate epoch * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review "--port flag". * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * Commented golangci. * staged stream sync v1.0 * fix protocol tests * fix spell * remove unused struct * fix rosetta test * add comments and refactor verify sig * add comments, remove extra function * add comment * refactor errors, rename metrics * refactor p2p host creation * fix initsync and host creation * fix short range hash chain * fix beacon node detection for p2p protocol * refactor stream peer cooldown and fix protocol beacon node field * refactor p2p host and routing * fix p2p discovery test issue * add MaxAdvertiseWaitTime to handle advertisements interval and address stream connection issue * terminal print the peer id and proto id * fix boot complete message when node is shut down * add new config option ( ForceReachabilityPublic ) to fix local-net consensus issue * fix self query issue * fix test NewDNSSyncingPeerProvider * [testnet] disable leader rotation * fix discovery issue for legacy sync * add watermark low/high options for p2p connection manager * add test for new conn manager flags * fix dedent * add comment to inform about p2p connection manager options * fix max height issue * add a separate log for get max height error * fix log * feat: triesInMemory flag * fix: panic if TriesInMemory is 1 to 2 * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed timers locks. * Removed fbft locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional logs. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * Fix formatting. * Rebased onto dev. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Fix review comment. * Go mod tidy. * Set to EpochTBD. * Fix tests. * [core] fix state handling of self destruct If a contract self destructs to self and then receives funds within the same transaction, it is possible for its stale state to be saved. This change removes that possibility by checking for deleted state objects before returning them. * Fixed race error. * rpc: add configurable http and `eth_call` timeout * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fixed code review. * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * staged stream sync v1.0 * refactor errors, rename metrics * fix p2p discovery test issue * add watermark low/high options for p2p connection manager * fix dedent * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Go mod tidy. * remove default timeouts * Rotate external leaders on non-beacon chains. * Fix nil panic. * Fixes. * Update singleton.go * evm: don't return extcode for validators Due to technical debt, validator information is stored in the code field of the address. The code field can be accessed in Solidity for an arbitrary address using `extcodesize`, `extcodehash`, and `extcodecopy` or helper commands (such as `address.code.Length`). The presence of this field is used by contract developers to (erroneously) deny smart contract access to other smart contracts (and therefore, validators). This PR fixes that oversight by returning the same values as other EOAs for known validator addresses. Obviously, it needs a hard fork that will be scheduled separately. * Fix context passing. * Clean up code. * Removed engine dependency. * Fix possible panic. * Clean up code. * Network type. * Fix tests. * Revert "Removed engine dependency." (#4392) * Revert "Fix tests." This reverts commit 597ba2d6f1ed54ff599b9d9b8940c7285ab1277a. * Revert "Network type." This reverts commit 5e1878aedca3989dc0f34161dae1833e43ca6a76. * Revert "Clean up code." This reverts commit 15885f4c9b9263746827172b4f4f56d0926d18e2. * Revert "Fix possible panic." This reverts commit 1a70d5eb66cdbf8a23791806b71a323eed320085. * Revert "Removed engine dependency." This reverts commit 8c2ff803f709f944cfc8b1278f35cf5b2cacf859. * gitignore the cache folder (#4389) * stable localnet with external validator (#4388) * stable localnet with external validator * ignore deploy config file comments * reduce node launched in localnet * update makefile * localnet configuration - add more fn * fix validator information command typo --------- Signed-off-by: dependabot[bot] Co-authored-by: Casey Gardiner <117784577+ONECasey@users.noreply.github.com> Co-authored-by: frozen <355847+Frozen@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Soph <35721420+sophoah@users.noreply.github.com> From 2c7dc046438bae5f95df6ef984245fb5bdacf178 Mon Sep 17 00:00:00 2001 From: Max <82761650+MaxMustermann2@users.noreply.github.com> Date: Wed, 26 Apr 2023 16:49:06 +0000 Subject: [PATCH 396/420] chore: merge `main` into `dev` (#4415) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Mainnet Release Candidate 2023.1.2 (#4376) * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * Bump github.com/aws/aws-sdk-go from 1.33.0 to 1.34.0 Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.33.0 to 1.34.0. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/v1.34.0/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.33.0...v1.34.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Bump github.com/ipld/go-ipld-prime from 0.9.0 to 0.19.0 Bumps [github.com/ipld/go-ipld-prime](https://github.com/ipld/go-ipld-prime) from 0.9.0 to 0.19.0. - [Release notes](https://github.com/ipld/go-ipld-prime/releases) - [Changelog](https://github.com/ipld/go-ipld-prime/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipld/go-ipld-prime/compare/v0.9.0...v0.19.0) --- updated-dependencies: - dependency-name: github.com/ipld/go-ipld-prime dependency-type: indirect ... Signed-off-by: dependabot[bot] * Bump golang.org/x/net from 0.3.0 to 0.7.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.3.0 to 0.7.0. - [Release notes](https://github.com/golang/net/releases) - [Commits](https://github.com/golang/net/compare/v0.3.0...v0.7.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] * Small fixes. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * activate epoch * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review "--port flag". * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * Commented golangci. * staged stream sync v1.0 * fix protocol tests * fix spell * remove unused struct * fix rosetta test * add comments and refactor verify sig * add comments, remove extra function * add comment * refactor errors, rename metrics * refactor p2p host creation * fix initsync and host creation * fix short range hash chain * fix beacon node detection for p2p protocol * refactor stream peer cooldown and fix protocol beacon node field * refactor p2p host and routing * fix p2p discovery test issue * add MaxAdvertiseWaitTime to handle advertisements interval and address stream connection issue * terminal print the peer id and proto id * fix boot complete message when node is shut down * add new config option ( ForceReachabilityPublic ) to fix local-net consensus issue * fix self query issue * fix test NewDNSSyncingPeerProvider * [testnet] disable leader rotation * fix discovery issue for legacy sync * add watermark low/high options for p2p connection manager * add test for new conn manager flags * fix dedent * add comment to inform about p2p connection manager options * fix max height issue * add a separate log for get max height error * fix log * feat: triesInMemory flag * fix: panic if TriesInMemory is 1 to 2 * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed timers locks. * Removed fbft locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional logs. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * Fix formatting. * Rebased onto dev. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Fix review comment. * Go mod tidy. * Set to EpochTBD. * Fix tests. * [core] fix state handling of self destruct If a contract self destructs to self and then receives funds within the same transaction, it is possible for its stale state to be saved. This change removes that possibility by checking for deleted state objects before returning them. * Fixed race error. * rpc: add configurable http and `eth_call` timeout * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fixed code review. * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * staged stream sync v1.0 * refactor errors, rename metrics * fix p2p discovery test issue * add watermark low/high options for p2p connection manager * fix dedent * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Go mod tidy. * remove default timeouts * Rotate external leaders on non-beacon chains. * Fix nil panic. * Fixes. * Update singleton.go * evm: don't return extcode for validators Due to technical debt, validator information is stored in the code field of the address. The code field can be accessed in Solidity for an arbitrary address using `extcodesize`, `extcodehash`, and `extcodecopy` or helper commands (such as `address.code.Length`). The presence of this field is used by contract developers to (erroneously) deny smart contract access to other smart contracts (and therefore, validators). This PR fixes that oversight by returning the same values as other EOAs for known validator addresses. Obviously, it needs a hard fork that will be scheduled separately. * Fix context passing. * Clean up code. * Removed engine dependency. * Fix possible panic. * Clean up code. * Network type. * Fix tests. * Revert "Removed engine dependency." (#4392) * Revert "Fix tests." This reverts commit 597ba2d6f1ed54ff599b9d9b8940c7285ab1277a. * Revert "Network type." This reverts commit 5e1878aedca3989dc0f34161dae1833e43ca6a76. * Revert "Clean up code." This reverts commit 15885f4c9b9263746827172b4f4f56d0926d18e2. * Revert "Fix possible panic." This reverts commit 1a70d5eb66cdbf8a23791806b71a323eed320085. * Revert "Removed engine dependency." This reverts commit 8c2ff803f709f944cfc8b1278f35cf5b2cacf859. * gitignore the cache folder (#4389) * stable localnet with external validator (#4388) * stable localnet with external validator * ignore deploy config file comments * reduce node launched in localnet * update makefile * localnet configuration - add more fn * fix validator information command typo --------- Signed-off-by: dependabot[bot] Co-authored-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Co-authored-by: frozen <355847+Frozen@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Soph <35721420+sophoah@users.noreply.github.com> * build: update pinned curl version (#4394) Per the Alpine Linux package repositories, the version for cURL included with v3.16 has changed to revision 6 * consensus: replace type assert with test (#4398) If `consensus.finalityCounter` does not have anything stored (for example in Syncing mode), the `Load()` returns an interface that cannot be automatically asserted to an `int64`. This results in the node crashing. This commit fixes that. * Turn pprof default on with local saved files (#3894) * Turn pprof default on with local saved files * [pprof] change interval from 600s to 3600s * Revert "Turn pprof default on with local saved files (#3894)" (#4400) This reverts commit 78d26d7910a58ded3bfe689b3de07ea28d95d7d5. --------- Signed-off-by: dependabot[bot] Co-authored-by: Casey Gardiner <117784577+ONECasey@users.noreply.github.com> Co-authored-by: frozen <355847+Frozen@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Soph <35721420+sophoah@users.noreply.github.com> Co-authored-by: Jacky Wang --- consensus/consensus_service.go | 6 ++++-- scripts/docker/Dockerfile | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 7ca863733f..e8d7e1645c 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -538,8 +538,10 @@ func (consensus *Consensus) StartFinalityCount() { // FinishFinalityCount calculate the current finality func (consensus *Consensus) FinishFinalityCount() { d := time.Now().UnixNano() - consensus.finality = (d - consensus.finalityCounter.Load().(int64)) / 1000000 - consensusFinalityHistogram.Observe(float64(consensus.finality)) + if prior, ok := consensus.finalityCounter.Load().(int64); ok { + consensus.finality = (d - prior) / 1000000 + consensusFinalityHistogram.Observe(float64(consensus.finality)) + } } // GetFinality returns the finality time in milliseconds of previous consensus diff --git a/scripts/docker/Dockerfile b/scripts/docker/Dockerfile index 5e85c98428..92b566ef47 100644 --- a/scripts/docker/Dockerfile +++ b/scripts/docker/Dockerfile @@ -39,7 +39,7 @@ ARG HARMONY_USER_GID=1000 ENV HARMONY_HOME=/harmony ENV HOME=${HARMONY_HOME} -RUN apk add --no-cache bash~=5.1.16-r2 bind-tools~=9.16.37-r0 tini~=0.19.0 curl==7.83.1-r5 sed~=4.8-r0 \ +RUN apk add --no-cache bash~=5.1.16-r2 bind-tools~=9.16.37-r0 tini~=0.19.0 curl==7.83.1-r6 sed~=4.8-r0 \ && rm -rf /var/cache/apk/* \ && addgroup -g ${HARMONY_USER_GID} ${HARMONY_USER} \ && adduser -u ${HARMONY_USER_UID} -G ${HARMONY_USER} --shell /sbin/nologin --no-create-home -D ${HARMONY_USER} \ From fb5cd83af483166ca903ef75363b7b2574194cd9 Mon Sep 17 00:00:00 2001 From: Max <82761650+MaxMustermann2@users.noreply.github.com> Date: Wed, 26 Apr 2023 16:54:12 +0000 Subject: [PATCH 397/420] internal/params: set validator code fix hard forks (#4413) * internal/params: schedule hard forks Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> * internal/params: set localnet fee collect epoch 2 Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> --------- Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> --- internal/params/config.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/internal/params/config.go b/internal/params/config.go index cc2f680d74..6b52249019 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -113,7 +113,7 @@ var ( LeaderRotationExternalNonBeaconLeaders: EpochTBD, LeaderRotationExternalBeaconLeaders: EpochTBD, FeeCollectEpoch: big.NewInt(1296), // 2023-04-28 07:14:20+00:00 - ValidatorCodeFixEpoch: EpochTBD, + ValidatorCodeFixEpoch: big.NewInt(1296), // 2023-04-28 07:14:20+00:00 } // PangaeaChainConfig contains the chain parameters for the Pangaea network. // All features except for CrossLink are enabled at launch. @@ -193,9 +193,9 @@ var ( SlotsLimitedEpoch: EpochTBD, // epoch to enable HIP-16 CrossShardXferPrecompileEpoch: big.NewInt(1), AllowlistEpoch: EpochTBD, - FeeCollectEpoch: big.NewInt(848), // 2023-04-28 04:33:33+00:00 LeaderRotationExternalNonBeaconLeaders: EpochTBD, LeaderRotationExternalBeaconLeaders: EpochTBD, + FeeCollectEpoch: big.NewInt(848), // 2023-04-28 04:33:33+00:00 ValidatorCodeFixEpoch: big.NewInt(848), } @@ -279,7 +279,7 @@ var ( LeaderRotationExternalNonBeaconLeaders: big.NewInt(5), LeaderRotationExternalBeaconLeaders: big.NewInt(6), FeeCollectEpoch: big.NewInt(2), - ValidatorCodeFixEpoch: EpochTBD, + ValidatorCodeFixEpoch: big.NewInt(2), } // AllProtocolChanges ... @@ -548,16 +548,25 @@ func (c *ChainConfig) mustValid() { panic(err) } } + // before staking epoch, fees were sent to coinbase require(c.FeeCollectEpoch.Cmp(c.StakingEpoch) >= 0, "must satisfy: FeeCollectEpoch >= StakingEpoch") + // obvious require(c.PreStakingEpoch.Cmp(c.StakingEpoch) < 0, "must satisfy: PreStakingEpoch < StakingEpoch") + // delegations can be made starting at PreStakingEpoch require(c.StakingPrecompileEpoch.Cmp(c.PreStakingEpoch) >= 0, "must satisfy: StakingPrecompileEpoch >= PreStakingEpoch") + // main functionality must come before the precompile + // see AcceptsCrossTx for why > and not >= require(c.CrossShardXferPrecompileEpoch.Cmp(c.CrossTxEpoch) > 0, "must satisfy: CrossShardXferPrecompileEpoch > CrossTxEpoch") + // the fix is applied only on the Solidity level, so you need eth compat require(c.ValidatorCodeFixEpoch.Cmp(c.EthCompatibleEpoch) >= 0, "must satisfy: ValidatorCodeFixEpoch >= EthCompatibleEpoch") + // we accept validator creation transactions starting at PreStakingEpoch + require(c.ValidatorCodeFixEpoch.Cmp(c.PreStakingEpoch) >= 0, + "must satisfy: ValidatorCodeFixEpoch >= PreStakingEpoch") } // IsEIP155 returns whether epoch is either equal to the EIP155 fork epoch or greater. From 525555dae05a8fccf04643a2058978e1c3447a40 Mon Sep 17 00:00:00 2001 From: Max <82761650+MaxMustermann2@users.noreply.github.com> Date: Tue, 2 May 2023 04:09:07 +0000 Subject: [PATCH 398/420] internal/params: schedule HIP28v2 + val code fix (#4416) Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> --- internal/params/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/params/config.go b/internal/params/config.go index 6b52249019..43e538243c 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -69,10 +69,10 @@ var ( SlotsLimitedEpoch: big.NewInt(999), // Around Fri, 27 May 2022 09:41:02 UTC with 2s block time CrossShardXferPrecompileEpoch: big.NewInt(1323), // Around Wed 8 Feb 11:30PM UTC AllowlistEpoch: EpochTBD, - FeeCollectEpoch: EpochTBD, LeaderRotationExternalNonBeaconLeaders: EpochTBD, LeaderRotationExternalBeaconLeaders: EpochTBD, - ValidatorCodeFixEpoch: EpochTBD, + FeeCollectEpoch: big.NewInt(1451), // 2023-05-17 04:02:00+00:00 + ValidatorCodeFixEpoch: big.NewInt(1451), } // TestnetChainConfig contains the chain parameters to run a node on the harmony test network. From c783659d1b569ebea7ae76f48749eaaaa27c041e Mon Sep 17 00:00:00 2001 From: Konstantin <355847+Frozen@users.noreply.github.com> Date: Mon, 1 May 2023 23:10:27 -0500 Subject: [PATCH 399/420] Dev fix conflicts. (#4417) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Mainnet Release Candidate 2023.1.2 (#4376) * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * Bump github.com/aws/aws-sdk-go from 1.33.0 to 1.34.0 Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.33.0 to 1.34.0. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/v1.34.0/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.33.0...v1.34.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Bump github.com/ipld/go-ipld-prime from 0.9.0 to 0.19.0 Bumps [github.com/ipld/go-ipld-prime](https://github.com/ipld/go-ipld-prime) from 0.9.0 to 0.19.0. - [Release notes](https://github.com/ipld/go-ipld-prime/releases) - [Changelog](https://github.com/ipld/go-ipld-prime/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipld/go-ipld-prime/compare/v0.9.0...v0.19.0) --- updated-dependencies: - dependency-name: github.com/ipld/go-ipld-prime dependency-type: indirect ... Signed-off-by: dependabot[bot] * Bump golang.org/x/net from 0.3.0 to 0.7.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.3.0 to 0.7.0. - [Release notes](https://github.com/golang/net/releases) - [Commits](https://github.com/golang/net/compare/v0.3.0...v0.7.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] * Small fixes. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * activate epoch * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review "--port flag". * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * Commented golangci. * staged stream sync v1.0 * fix protocol tests * fix spell * remove unused struct * fix rosetta test * add comments and refactor verify sig * add comments, remove extra function * add comment * refactor errors, rename metrics * refactor p2p host creation * fix initsync and host creation * fix short range hash chain * fix beacon node detection for p2p protocol * refactor stream peer cooldown and fix protocol beacon node field * refactor p2p host and routing * fix p2p discovery test issue * add MaxAdvertiseWaitTime to handle advertisements interval and address stream connection issue * terminal print the peer id and proto id * fix boot complete message when node is shut down * add new config option ( ForceReachabilityPublic ) to fix local-net consensus issue * fix self query issue * fix test NewDNSSyncingPeerProvider * [testnet] disable leader rotation * fix discovery issue for legacy sync * add watermark low/high options for p2p connection manager * add test for new conn manager flags * fix dedent * add comment to inform about p2p connection manager options * fix max height issue * add a separate log for get max height error * fix log * feat: triesInMemory flag * fix: panic if TriesInMemory is 1 to 2 * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed timers locks. * Removed fbft locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional logs. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * Fix formatting. * Rebased onto dev. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Fix review comment. * Go mod tidy. * Set to EpochTBD. * Fix tests. * [core] fix state handling of self destruct If a contract self destructs to self and then receives funds within the same transaction, it is possible for its stale state to be saved. This change removes that possibility by checking for deleted state objects before returning them. * Fixed race error. * rpc: add configurable http and `eth_call` timeout * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fixed code review. * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * staged stream sync v1.0 * refactor errors, rename metrics * fix p2p discovery test issue * add watermark low/high options for p2p connection manager * fix dedent * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Go mod tidy. * remove default timeouts * Rotate external leaders on non-beacon chains. * Fix nil panic. * Fixes. * Update singleton.go * evm: don't return extcode for validators Due to technical debt, validator information is stored in the code field of the address. The code field can be accessed in Solidity for an arbitrary address using `extcodesize`, `extcodehash`, and `extcodecopy` or helper commands (such as `address.code.Length`). The presence of this field is used by contract developers to (erroneously) deny smart contract access to other smart contracts (and therefore, validators). This PR fixes that oversight by returning the same values as other EOAs for known validator addresses. Obviously, it needs a hard fork that will be scheduled separately. * Fix context passing. * Clean up code. * Removed engine dependency. * Fix possible panic. * Clean up code. * Network type. * Fix tests. * Revert "Removed engine dependency." (#4392) * Revert "Fix tests." This reverts commit 597ba2d6f1ed54ff599b9d9b8940c7285ab1277a. * Revert "Network type." This reverts commit 5e1878aedca3989dc0f34161dae1833e43ca6a76. * Revert "Clean up code." This reverts commit 15885f4c9b9263746827172b4f4f56d0926d18e2. * Revert "Fix possible panic." This reverts commit 1a70d5eb66cdbf8a23791806b71a323eed320085. * Revert "Removed engine dependency." This reverts commit 8c2ff803f709f944cfc8b1278f35cf5b2cacf859. * gitignore the cache folder (#4389) * stable localnet with external validator (#4388) * stable localnet with external validator * ignore deploy config file comments * reduce node launched in localnet * update makefile * localnet configuration - add more fn * fix validator information command typo --------- Signed-off-by: dependabot[bot] Co-authored-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Co-authored-by: frozen <355847+Frozen@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Soph <35721420+sophoah@users.noreply.github.com> * build: update pinned curl version (#4394) Per the Alpine Linux package repositories, the version for cURL included with v3.16 has changed to revision 6 * consensus: replace type assert with test (#4398) If `consensus.finalityCounter` does not have anything stored (for example in Syncing mode), the `Load()` returns an interface that cannot be automatically asserted to an `int64`. This results in the node crashing. This commit fixes that. * Turn pprof default on with local saved files (#3894) * Turn pprof default on with local saved files * [pprof] change interval from 600s to 3600s * Revert "Turn pprof default on with local saved files (#3894)" (#4400) This reverts commit 78d26d7910a58ded3bfe689b3de07ea28d95d7d5. * go mod tidy. * Increased wait time. --------- Signed-off-by: dependabot[bot] Co-authored-by: Casey Gardiner <117784577+ONECasey@users.noreply.github.com> Co-authored-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Soph <35721420+sophoah@users.noreply.github.com> Co-authored-by: Jacky Wang --- core/tx_pool_test.go | 2 +- go.mod | 2 +- go.sum | 16 ++++++---------- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 9e1bba296a..9b126eac90 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -1086,7 +1086,7 @@ func testTransactionQueueTimeLimiting(t *testing.T, nolocals bool) { t.Fatalf("pool internal state corrupted: %v", err) } // Wait a bit for eviction to run and clean up any leftovers, and ensure only the local remains - time.Sleep(2 * config.Lifetime) + time.Sleep(4 * config.Lifetime) pending, queued = pool.Stats() if pending != 0 { diff --git a/go.mod b/go.mod index 2a2cabbbfc..4dd98958b3 100644 --- a/go.mod +++ b/go.mod @@ -68,6 +68,7 @@ require ( require ( github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b + github.com/holiman/bloomfilter/v2 v2.0.3 github.com/ledgerwatch/erigon-lib v0.0.0-20221218022306-0f8fdd40c2db github.com/ledgerwatch/log/v3 v3.6.0 github.com/libp2p/go-libp2p-core v0.20.1 @@ -151,7 +152,6 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/big v0.0.0-20221017200358-a027dc42d04e // indirect - github.com/holiman/bloomfilter/v2 v2.0.3 // indirect github.com/holiman/uint256 v1.2.1 // indirect github.com/huin/goupnp v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect diff --git a/go.sum b/go.sum index 81898e40b2..8b86a286c2 100644 --- a/go.sum +++ b/go.sum @@ -114,6 +114,8 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go v1.34.0 h1:brux2dRrlwCF5JhTL7MUT3WUwo9zfDHZZp3+g3Mvlmo= +github.com/aws/aws-sdk-go v1.34.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y= github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo= @@ -125,8 +127,6 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxq github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= -github.com/aws/aws-sdk-go v1.34.0 h1:brux2dRrlwCF5JhTL7MUT3WUwo9zfDHZZp3+g3Mvlmo= -github.com/aws/aws-sdk-go v1.34.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -689,7 +689,6 @@ github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19y github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= -github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= @@ -710,6 +709,8 @@ github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscw github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= +github.com/ipld/go-ipld-prime v0.19.0 h1:5axC7rJmPc17Emw6TelxGwnzALk0PdupZ2oj2roDj04= +github.com/ipld/go-ipld-prime v0.19.0/go.mod h1:Q9j3BaVXwaA3o5JUDNvptDDr/x8+F7FG6XJ8WI3ILg4= github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= github.com/iris-contrib/httpexpect/v2 v2.3.1/go.mod h1:ICTf89VBKSD3KB0fsyyHviKF8G8hyepP0dOXJPWz3T0= @@ -719,9 +720,6 @@ github.com/iris-contrib/jade v1.1.4/go.mod h1:EDqR+ur9piDl6DUgs6qRrlfzmlx/D5Uybo github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/iris-contrib/schema v0.0.6/go.mod h1:iYszG0IOsuIsfzjymw1kMzTL8YQcCWlm65f3wX8J5iA= -github.com/ipld/go-ipld-prime v0.19.0 h1:5axC7rJmPc17Emw6TelxGwnzALk0PdupZ2oj2roDj04= -github.com/ipld/go-ipld-prime v0.19.0/go.mod h1:Q9j3BaVXwaA3o5JUDNvptDDr/x8+F7FG6XJ8WI3ILg4= -github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= @@ -1127,10 +1125,10 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e h1:ZOcivgkkFRnjfoTcGsDq3UQYiBmekwLA+qg0OjyB/ls= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -1512,7 +1510,6 @@ golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= @@ -1765,7 +1762,6 @@ golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From 620385730b743e628b4404e6f8a89752633ba22e Mon Sep 17 00:00:00 2001 From: Casey Gardiner <117784577+ONECasey@users.noreply.github.com> Date: Tue, 2 May 2023 10:54:18 -0700 Subject: [PATCH 400/420] Remove dockerfile hardcoded version numbers (#4418) --- scripts/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/docker/Dockerfile b/scripts/docker/Dockerfile index 92b566ef47..197117c185 100644 --- a/scripts/docker/Dockerfile +++ b/scripts/docker/Dockerfile @@ -39,7 +39,7 @@ ARG HARMONY_USER_GID=1000 ENV HARMONY_HOME=/harmony ENV HOME=${HARMONY_HOME} -RUN apk add --no-cache bash~=5.1.16-r2 bind-tools~=9.16.37-r0 tini~=0.19.0 curl==7.83.1-r6 sed~=4.8-r0 \ +RUN apk add --no-cache bash bind-tools tini curl sed \ && rm -rf /var/cache/apk/* \ && addgroup -g ${HARMONY_USER_GID} ${HARMONY_USER} \ && adduser -u ${HARMONY_USER_UID} -G ${HARMONY_USER} --shell /sbin/nologin --no-create-home -D ${HARMONY_USER} \ From 31d54dc19599f75fe72702db94ea95fc65a85461 Mon Sep 17 00:00:00 2001 From: Konstantin <355847+Frozen@users.noreply.github.com> Date: Fri, 5 May 2023 01:53:29 -0500 Subject: [PATCH 401/420] Fix concurrent map access. (#4421) --- consensus/consensus_v2.go | 21 +++++++++++++++++++++ consensus/fbft_log.go | 10 +++++----- node/node_explorer.go | 8 ++++---- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 8e72896c3e..41d5b2d261 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -915,3 +915,24 @@ func (consensus *Consensus) ValidateVdfAndProof(headerObj *block.Header) bool { return true } + +// DeleteBlocksLessThan deletes blocks less than given block number +func (consensus *Consensus) DeleteBlocksLessThan(number uint64) { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + consensus.FBFTLog.deleteBlocksLessThan(number) +} + +// DeleteMessagesLessThan deletes messages less than given block number. +func (consensus *Consensus) DeleteMessagesLessThan(number uint64) { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + consensus.FBFTLog.deleteMessagesLessThan(number) +} + +// DeleteBlockByNumber deletes block by given block number. +func (consensus *Consensus) DeleteBlockByNumber(number uint64) { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + consensus.FBFTLog.deleteBlockByNumber(number) +} diff --git a/consensus/fbft_log.go b/consensus/fbft_log.go index ba37451038..5a4fe21889 100644 --- a/consensus/fbft_log.go +++ b/consensus/fbft_log.go @@ -147,7 +147,7 @@ func (log *FBFTLog) GetBlocksByNumber(number uint64) []*types.Block { } // DeleteBlocksLessThan deletes blocks less than given block number -func (log *FBFTLog) DeleteBlocksLessThan(number uint64) { +func (log *FBFTLog) deleteBlocksLessThan(number uint64) { for h, block := range log.blocks { if block.NumberU64() < number { delete(log.blocks, h) @@ -157,7 +157,7 @@ func (log *FBFTLog) DeleteBlocksLessThan(number uint64) { } // DeleteBlockByNumber deletes block of specific number -func (log *FBFTLog) DeleteBlockByNumber(number uint64) { +func (log *FBFTLog) deleteBlockByNumber(number uint64) { for h, block := range log.blocks { if block.NumberU64() == number { delete(log.blocks, h) @@ -167,7 +167,7 @@ func (log *FBFTLog) DeleteBlockByNumber(number uint64) { } // DeleteMessagesLessThan deletes messages less than given block number -func (log *FBFTLog) DeleteMessagesLessThan(number uint64) { +func (log *FBFTLog) deleteMessagesLessThan(number uint64) { for h, msg := range log.messages { if msg.BlockNum < number { delete(log.messages, h) @@ -357,6 +357,6 @@ func (log *FBFTLog) GetCommittedBlockAndMsgsFromNumber(bn uint64, logger *zerolo // PruneCacheBeforeBlock prune all blocks before bn func (log *FBFTLog) PruneCacheBeforeBlock(bn uint64) { - log.DeleteBlocksLessThan(bn - 1) - log.DeleteMessagesLessThan(bn - 1) + log.deleteBlocksLessThan(bn - 1) + log.deleteMessagesLessThan(bn - 1) } diff --git a/node/node_explorer.go b/node/node_explorer.go index 0d3f31f28b..d1c78d612a 100644 --- a/node/node_explorer.go +++ b/node/node_explorer.go @@ -148,7 +148,7 @@ func (node *Node) TraceLoopForExplorer() { // AddNewBlockForExplorer add new block for explorer. func (node *Node) AddNewBlockForExplorer(block *types.Block) { if node.HarmonyConfig.General.RunElasticMode && node.HarmonyConfig.TiKV.Role == tikv.RoleReader { - node.Consensus.FBFTLog.DeleteBlockByNumber(block.NumberU64()) + node.Consensus.DeleteBlockByNumber(block.NumberU64()) return } @@ -159,7 +159,7 @@ func (node *Node) AddNewBlockForExplorer(block *types.Block) { node.Consensus.UpdateConsensusInformation() } // Clean up the blocks to avoid OOM. - node.Consensus.FBFTLog.DeleteBlockByNumber(block.NumberU64()) + node.Consensus.DeleteBlockByNumber(block.NumberU64()) // if in tikv mode, only master writer node need dump all explorer block if !node.HarmonyConfig.General.RunElasticMode || node.Blockchain().IsTikvWriterMaster() { @@ -216,8 +216,8 @@ func (node *Node) commitBlockForExplorer(block *types.Block) { curNum := block.NumberU64() if curNum-100 > 0 { - node.Consensus.FBFTLog.DeleteBlocksLessThan(curNum - 100) - node.Consensus.FBFTLog.DeleteMessagesLessThan(curNum - 100) + node.Consensus.DeleteBlocksLessThan(curNum - 100) + node.Consensus.DeleteMessagesLessThan(curNum - 100) } } From f317e536c05517789860aac9ae37167a7eef4249 Mon Sep 17 00:00:00 2001 From: Gheis Mohammadi Date: Thu, 11 May 2023 14:02:48 +0800 Subject: [PATCH 402/420] add inspectdb to cli (#4426) * add inspect db to cli * remove extra comment --- cmd/harmony/inspectdb.go | 59 ++++++++++++++++++++++++++++++++++++++++ cmd/harmony/main.go | 4 +++ core/rawdb/database.go | 30 ++++++++------------ 3 files changed, 75 insertions(+), 18 deletions(-) create mode 100644 cmd/harmony/inspectdb.go diff --git a/cmd/harmony/inspectdb.go b/cmd/harmony/inspectdb.go new file mode 100644 index 0000000000..89f8c28271 --- /dev/null +++ b/cmd/harmony/inspectdb.go @@ -0,0 +1,59 @@ +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/harmony-one/harmony/core/rawdb" + "github.com/harmony-one/harmony/internal/cli" +) + +var prefixFlag = cli.StringFlag{ + Name: "prefix", + Shorthand: "p", + Usage: "key prefix", + DefValue: "", +} + +var startKeyFlag = cli.StringFlag{ + Name: "start_key", + Shorthand: "s", + Usage: "start key", + DefValue: "", +} + +var inspectDBCmd = &cobra.Command{ + Use: "inspectdb srcdb prefix startKey", + Short: "inspect a db.", + Long: "inspect a db.", + Example: "harmony inspectdb /srcDir/harmony_db_0", + Args: cobra.RangeArgs(1, 3), + Run: func(cmd *cobra.Command, args []string) { + srcDBDir := args[0] + prefix := cli.GetStringFlagValue(cmd, prefixFlag) + startKey := cli.GetStringFlagValue(cmd, startKeyFlag) + fmt.Println("db path: ", srcDBDir) + inspectDB(srcDBDir, prefix, startKey) + os.Exit(0) + }, +} + +func registerInspectionFlags() error { + return cli.RegisterFlags(inspectDBCmd, []cli.Flag{prefixFlag, startKeyFlag}) + +} + +func inspectDB(srcDBDir, prefix, startKey string) { + fmt.Println("===inspectDB===") + srcDB, err := rawdb.NewLevelDBDatabase(srcDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "", false) + if err != nil { + fmt.Println("open src db error:", err) + os.Exit(-1) + } + + rawdb.InspectDatabase(srcDB, []byte(prefix), []byte(startKey)) + + fmt.Println("database inspection completed!") +} diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 88da5f7513..94038749d4 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -111,6 +111,7 @@ func init() { rootCmd.AddCommand(versionCmd) rootCmd.AddCommand(dumpConfigLegacyCmd) rootCmd.AddCommand(dumpDBCmd) + rootCmd.AddCommand(inspectDBCmd) if err := registerRootCmdFlags(); err != nil { os.Exit(2) @@ -121,6 +122,9 @@ func init() { if err := registerDumpDBFlags(); err != nil { os.Exit(2) } + if err := registerInspectionFlags(); err != nil { + os.Exit(2) + } } func main() { diff --git a/core/rawdb/database.go b/core/rawdb/database.go index c1421b5073..20fac099f4 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -23,7 +23,6 @@ import ( "os" "path" "path/filepath" - "strings" "time" "github.com/ethereum/go-ethereum/common" @@ -297,7 +296,6 @@ func (s *stat) Count() string { // InspectDatabase traverses the entire database and checks the size // of all different categories of data. -// This function is NOT used, just ported over from the Ethereum func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { it := db.NewIterator(keyPrefix, keyStart) defer it.Release() @@ -332,6 +330,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { // Meta- and unaccounted data metadata stat unaccounted stat + zeroval stat // Totals total common.StorageSize @@ -342,6 +341,9 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { key = it.Key() size = common.StorageSize(len(key) + len(it.Value())) ) + if len(it.Value()) == 0 { + zeroval.Add(1) + } total += size switch { case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): @@ -433,31 +435,23 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { {"Key-Value store", "Beacon sync headers", beaconHeaders.Size(), beaconHeaders.Count()}, {"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()}, {"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()}, + {"Key-Value store", "Zero value keys", zeroval.Size(), zeroval.Count()}, + {"Key-Value store", "Unaccounted", unaccounted.Size(), unaccounted.Count()}, {"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()}, {"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()}, } - // Inspect all registered append-only file store then. - ancients, err := inspectFreezers(db) - if err != nil { - return err - } - for _, ancient := range ancients { - for _, table := range ancient.sizes { - stats = append(stats, []string{ - fmt.Sprintf("Ancient store (%s)", strings.Title(ancient.name)), - strings.Title(table.name), - table.size.String(), - fmt.Sprintf("%d", ancient.count()), - }) - } - total += ancient.size() - } table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"Database", "Category", "Size", "Items"}) table.SetFooter([]string{"", "Total", total.String(), " "}) table.AppendBulk(stats) table.Render() + if zeroval.count > 0 { + utils.Logger().Error(). + Interface("count", zeroval). + Msg("Database contains zero value keys") + } + if unaccounted.size > 0 { utils.Logger().Error(). Interface("size", unaccounted.size). From dae82158a4e6ae1d23b4edddfd4cb3c968b53e23 Mon Sep 17 00:00:00 2001 From: Gheis Mohammadi Date: Fri, 12 May 2023 00:41:25 +0800 Subject: [PATCH 403/420] fix beacon-ness in legacy sync (#4428) * fix beacon-ness * use enum to identify beacon shard --- api/service/legacysync/epoch_syncing.go | 2 +- node/node_syncing.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/api/service/legacysync/epoch_syncing.go b/api/service/legacysync/epoch_syncing.go index eefca9a5cb..e4453cb69c 100644 --- a/api/service/legacysync/epoch_syncing.go +++ b/api/service/legacysync/epoch_syncing.go @@ -92,7 +92,7 @@ func (ss *EpochSync) SyncLoop(bc core.BlockChain, consensus *consensus.Consensus } func syncLoop(bc core.BlockChain, syncConfig *SyncConfig) (timeout int) { - isBeacon := bc.ShardID() == 0 + isBeacon := bc.ShardID() == shard.BeaconChainShardID maxHeight, errMaxHeight := getMaxPeerHeight(syncConfig) if errMaxHeight != nil { utils.Logger().Info(). diff --git a/node/node_syncing.go b/node/node_syncing.go index 05ddd4b6b7..c33dd02c22 100644 --- a/node/node_syncing.go +++ b/node/node_syncing.go @@ -314,7 +314,8 @@ func (node *Node) doSync(bc core.BlockChain, worker *worker.Worker, willJoinCons if willJoinConsensus { node.Consensus.BlocksNotSynchronized() } - syncInstance.SyncLoop(bc, worker, false, node.Consensus, legacysync.LoopMinTime) + isBeacon := bc.ShardID() == shard.BeaconChainShardID + syncInstance.SyncLoop(bc, worker, isBeacon, node.Consensus, legacysync.LoopMinTime) if willJoinConsensus { node.IsSynchronized.Set() node.Consensus.BlocksSynchronized() From ef171ef30176d6d8c2f72ddf91446ab799b7b2bd Mon Sep 17 00:00:00 2001 From: Konstantin <355847+Frozen@users.noreply.github.com> Date: Tue, 16 May 2023 12:48:10 -0500 Subject: [PATCH 404/420] Use Header instead Block. (#4424) * Block to Header. * Block to Header. * Block to Header. --- core/block_validator.go | 2 +- internal/chain/reward.go | 4 ++-- node/worker/worker.go | 6 +++--- test/chain/chain/chain_makers.go | 25 ++++++------------------- test/chain/main.go | 2 +- 5 files changed, 13 insertions(+), 26 deletions(-) diff --git a/core/block_validator.go b/core/block_validator.go index b1126f3972..fe5ff0f44a 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -154,7 +154,7 @@ func (v *BlockValidator) ValidateHeaders(chain []*types.Block) (chan<- struct{}, // to keep the baseline gas above the provided floor, and increase it towards the // ceil if the blocks are full. If the ceil is exceeded, it will always decrease // the gas allowance. -func CalcGasLimit(parent *types.Block, gasFloor, gasCeil uint64) uint64 { +func CalcGasLimit(parent *block.Header, gasFloor, gasCeil uint64) uint64 { // contrib = (parentGasUsed * 3 / 2) / 1024 contrib := (parent.GasUsed() + parent.GasUsed()/2) / params.GasLimitBoundDivisor diff --git a/internal/chain/reward.go b/internal/chain/reward.go index 664b53bcad..f06556412f 100644 --- a/internal/chain/reward.go +++ b/internal/chain/reward.go @@ -138,7 +138,7 @@ func lookupDelegatorShares( // Handle block rewards during pre-staking era func accumulateRewardsAndCountSigsBeforeStaking( bc engine.ChainReader, state *state.DB, - header *block.Header, beaconChain engine.ChainReader, sigsReady chan bool, + header *block.Header, sigsReady chan bool, ) (reward.Reader, error) { parentHeader := bc.GetHeaderByHash(header.ParentHash()) @@ -252,7 +252,7 @@ func AccumulateRewardsAndCountSigs( // Pre-staking era if !bc.Config().IsStaking(epoch) { - return accumulateRewardsAndCountSigsBeforeStaking(bc, state, header, beaconChain, sigsReady) + return accumulateRewardsAndCountSigsBeforeStaking(bc, state, header, sigsReady) } // Rewards are accumulated only in the beaconchain, so just wait for commit sigs and return. diff --git a/node/worker/worker.go b/node/worker/worker.go index 1a711bc193..5a4234229a 100644 --- a/node/worker/worker.go +++ b/node/worker/worker.go @@ -300,7 +300,7 @@ func (w *Worker) CommitReceipts(receiptsList []*types.CXReceiptsProof) error { // UpdateCurrent updates the current environment with the current state and header. func (w *Worker) UpdateCurrent() error { - parent := w.chain.CurrentBlock() + parent := w.chain.CurrentHeader() num := parent.Number() timestamp := time.Now().Unix() @@ -321,7 +321,7 @@ func (w *Worker) GetCurrentHeader() *block.Header { } // makeCurrent creates a new environment for the current cycle. -func (w *Worker) makeCurrent(parent *types.Block, header *block.Header) error { +func (w *Worker) makeCurrent(parent *block.Header, header *block.Header) error { state, err := w.chain.StateAt(parent.Root()) if err != nil { return err @@ -586,7 +586,7 @@ func New( worker.gasFloor = 80000000 worker.gasCeil = 120000000 - parent := worker.chain.CurrentBlock() + parent := worker.chain.CurrentBlock().Header() num := parent.Number() timestamp := time.Now().Unix() diff --git a/test/chain/chain/chain_makers.go b/test/chain/chain/chain_makers.go index a902fc8441..52c66b2692 100644 --- a/test/chain/chain/chain_makers.go +++ b/test/chain/chain/chain_makers.go @@ -39,7 +39,7 @@ import ( // See GenerateChain for a detailed explanation. type BlockGen struct { i int - parent *types.Block + parent *block.Header chain []*types.Block factory blockfactory.Factory header *block.Header @@ -140,19 +140,6 @@ func (b *BlockGen) AddUncle(h *block.Header) { b.uncles = append(b.uncles, h) } -// PrevBlock returns a previously generated block by number. It panics if -// num is greater or equal to the number of the block being generated. -// For index -1, PrevBlock returns the parent block given to GenerateChain. -func (b *BlockGen) PrevBlock(index int) *types.Block { - if index >= b.i { - panic(fmt.Errorf("block index %d out of range (%d,%d)", index, -1, b.i)) - } - if index == -1 { - return b.parent - } - return b.chain[index] -} - // GenerateChain creates a chain of n blocks. The first block's // parent will be the provided parent. db is used to store // intermediate states and should contain the parent's state trie. @@ -166,7 +153,7 @@ func (b *BlockGen) PrevBlock(index int) *types.Block { // values. Inserting them into BlockChain requires use of FakePow or // a similar non-validating proof of work implementation. func GenerateChain( - config *params.ChainConfig, parent *types.Block, + config *params.ChainConfig, parent *block.Header, engine consensus_engine.Engine, db ethdb.Database, n int, gen func(int, *BlockGen), @@ -177,7 +164,7 @@ func GenerateChain( factory := blockfactory.NewFactory(config) blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n) chainreader := &fakeChainReader{config: config} - genblock := func(i int, parent *types.Block, statedb *state.DB) (*types.Block, types.Receipts) { + genblock := func(i int, parent *block.Header, statedb *state.DB) (*types.Block, types.Receipts) { b := &BlockGen{ i: i, chain: blocks, @@ -187,7 +174,7 @@ func GenerateChain( factory: factory, engine: engine, } - b.header = makeHeader(chainreader, parent, statedb, b.engine, factory) + b.header = makeHeader(chainreader, parent, statedb, factory) // Execute any user modifications to the block if gen != nil { @@ -223,12 +210,12 @@ func GenerateChain( block, receipt := genblock(i, parent, statedb) blocks[i] = block receipts[i] = receipt - parent = block + parent = block.Header() } return blocks, receipts } -func makeHeader(chain consensus_engine.ChainReader, parent *types.Block, state *state.DB, engine consensus_engine.Engine, factory blockfactory.Factory) *block.Header { +func makeHeader(chain consensus_engine.ChainReader, parent *block.Header, state *state.DB, factory blockfactory.Factory) *block.Header { var time *big.Int if parent.Time() == nil { time = big.NewInt(10) diff --git a/test/chain/main.go b/test/chain/main.go index fd1a74e737..656e90c949 100644 --- a/test/chain/main.go +++ b/test/chain/main.go @@ -223,7 +223,7 @@ func main() { //// Generate a small n-block chain and an uncle block for it n := 3 if n > 0 { - blocks, _ := chain2.GenerateChain(chainConfig, genesis, chain.Engine(), database, n, func(i int, gen *chain2.BlockGen) { + blocks, _ := chain2.GenerateChain(chainConfig, genesis.Header(), chain.Engine(), database, n, func(i int, gen *chain2.BlockGen) { gen.SetCoinbase(FaucetAddress) gen.SetShardID(0) gen.AddTx(pendingTxs[i].(*types.Transaction)) From d2a34630efc7b521210920f38458c89717ecc532 Mon Sep 17 00:00:00 2001 From: Soph <35721420+sophoah@users.noreply.github.com> Date: Sat, 20 May 2023 08:57:53 +0700 Subject: [PATCH 405/420] Sync dev with main branch ( v2023.2.1 hotfix ) (#4435) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Release Candidate 2023.2.0 ( dev -> main ) (#4399) * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * activate epoch * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review "--port flag". * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * Commented golangci. * staged stream sync v1.0 * fix protocol tests * fix spell * remove unused struct * fix rosetta test * add comments and refactor verify sig * add comments, remove extra function * add comment * refactor errors, rename metrics * refactor p2p host creation * fix initsync and host creation * fix short range hash chain * fix beacon node detection for p2p protocol * refactor stream peer cooldown and fix protocol beacon node field * refactor p2p host and routing * fix p2p discovery test issue * add MaxAdvertiseWaitTime to handle advertisements interval and address stream connection issue * terminal print the peer id and proto id * fix boot complete message when node is shut down * add new config option ( ForceReachabilityPublic ) to fix local-net consensus issue * fix self query issue * fix test NewDNSSyncingPeerProvider * [testnet] disable leader rotation * fix discovery issue for legacy sync * add watermark low/high options for p2p connection manager * add test for new conn manager flags * fix dedent * add comment to inform about p2p connection manager options * fix max height issue * add a separate log for get max height error * fix log * feat: triesInMemory flag * fix: panic if TriesInMemory is 1 to 2 * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed timers locks. * Removed fbft locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional logs. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * Fix formatting. * Rebased onto dev. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Fix review comment. * Go mod tidy. * Set to EpochTBD. * Fix tests. * [core] fix state handling of self destruct If a contract self destructs to self and then receives funds within the same transaction, it is possible for its stale state to be saved. This change removes that possibility by checking for deleted state objects before returning them. * Fixed race error. * rpc: add configurable http and `eth_call` timeout * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fixed code review. * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * staged stream sync v1.0 * refactor errors, rename metrics * fix p2p discovery test issue * add watermark low/high options for p2p connection manager * fix dedent * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Go mod tidy. * remove default timeouts * Rotate external leaders on non-beacon chains. * Fix nil panic. * Fixes. * Update singleton.go * evm: don't return extcode for validators Due to technical debt, validator information is stored in the code field of the address. The code field can be accessed in Solidity for an arbitrary address using `extcodesize`, `extcodehash`, and `extcodecopy` or helper commands (such as `address.code.Length`). The presence of this field is used by contract developers to (erroneously) deny smart contract access to other smart contracts (and therefore, validators). This PR fixes that oversight by returning the same values as other EOAs for known validator addresses. Obviously, it needs a hard fork that will be scheduled separately. * Fix context passing. * Clean up code. * Removed engine dependency. * Fix possible panic. * Clean up code. * Network type. * Fix tests. * Revert "Removed engine dependency." (#4392) * Revert "Fix tests." This reverts commit 597ba2d6f1ed54ff599b9d9b8940c7285ab1277a. * Revert "Network type." This reverts commit 5e1878aedca3989dc0f34161dae1833e43ca6a76. * Revert "Clean up code." This reverts commit 15885f4c9b9263746827172b4f4f56d0926d18e2. * Revert "Fix possible panic." This reverts commit 1a70d5eb66cdbf8a23791806b71a323eed320085. * Revert "Removed engine dependency." This reverts commit 8c2ff803f709f944cfc8b1278f35cf5b2cacf859. * gitignore the cache folder (#4389) * stable localnet with external validator (#4388) * stable localnet with external validator * ignore deploy config file comments * reduce node launched in localnet * update makefile * localnet configuration - add more fn * fix validator information command typo * Configurable tx pool. (#4240) * AccountQueue & GlobalQueue. * Lifetime duration. * [pool] make flags configurable * [pool] use 4096 as default `GlobalSlots` * [rosetta] update default values of tx pool * [test] update value to default * PriceLimit and PriceBump. * Fix tests. * Fix price limit & bump. * Updated, fixed migrate version and tests. * Rebased. * Fix go toml version. --------- Co-authored-by: Konstantin Co-authored-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> * Upgrade rawdb and statedb codes to add the latest functionalities of ethdb (#4374) * added bloom filter * upgrade rawdb and statedb * change var name and remove extra comments * return back fake storage in case if we need it for test later * add the previous change back * remove some extra entries from go.mod * fix WritePreimages to use batch * mark unused functions which are ported over from eth --------- Co-authored-by: Casey Gardiner <117784577+ONECasey@users.noreply.github.com> * update all ethereum rawdb pointers to use harmony rawdb (#4395) * Fix reduce node dependencies. (#4379) * Fix. * Fix. * Fix pool init. * Clean up. * add prefix for contract code (#4397) * Rotate external validators for non-beacon shards. (#4373) * Rotate only non beacon shards. * Rotate all shards, but only hmy validators for beacon. * Fix type. * Revert "Fix type." This reverts commit 0a8b506c763d9f8609abff7395ba32b18e43b149. * Revert "Rotate all shards, but only hmy validators for beacon." This reverts commit 70b09e2de81aa2cbffae3ccdfd4e334e7d938759. * Fixed failed test. * Revert "Revert "Rotate all shards, but only hmy validators for beacon."" This reverts commit 66cfaa9817488be60ed5b5cfee1fe833ede237c8. * Frequency by slots count. * Fix config. * First validator produce rest blocks. * Updated. * Add lock. * Add prefix for validator wrapper (#4402) * add separate prefix for validator wrapper * update comments * make read/write backward compatible * add validator codes to stats * goimports * goimports accessor_state * add snapshot feature to state db (#4406) * Typed cache & Node cleanup. (#4409) * Channels usage through methods. * Fix retry count. Removed proposedBlock. * keysToAddrs rewritten to lrucache. * core, internal/configs: HIP28-v2 fee collection (#4410) * core, internal/configs: HIP28-v2 fee collection Based on the Snapshot vote that has passed, collect 50% of the fee to a community maintained account and the remainder to an account used to pay for infrastructure costs. Note that these accounts need to be decided and set in the code at this moment, and the feature needs to be activated by setting the `FeeCollectEpoch` of the `ChainConfig` object. The implementation for devnet is a bit different than compared to others because the feature was activated on devnet with 100% collection to an account. I have handled this case separately in `devnet.go`. * test: add test for StateTransition.ApplyMessage The objective of this unit test is to check that the fees of a transaction are appropriately credited to the fee collectors that are set. This means, for a transaction of 21,000 gas limit and 100 gwei gas price, two equal fee collectors get 10,500 * 100 gwei each. In addition, to be clear that the refund mechanism (in case a transaction with extra gas comes in) works, the tested transaction has a 50,000 gas limit of which only 21,000 gas limit is actually consumed. * sharding/config: clarify local fee collector pk * sharding/config: set testnet fee collector same as devnet * test: add test for truncated fee distribution * sharding/config: set fee collector addresses * test: hardcode the expected fee collected * goimports * params/config: set testnet fee collect epoch Schedule testnet hard fork epoch to be 1296, which begins around the time 2023-04-28 07:14:20+00:00 * params/config: schedule devnee fee collection Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> * Minor: removed time.Sleep from tests. (#4412) * Provide current time as argument. * Fix test. * Fix naming. * Mainnet Release Candidate 2023.1.2 (#4376) (#4414) * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * Bump github.com/aws/aws-sdk-go from 1.33.0 to 1.34.0 Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.33.0 to 1.34.0. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/v1.34.0/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.33.0...v1.34.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production ... * Bump github.com/ipld/go-ipld-prime from 0.9.0 to 0.19.0 Bumps [github.com/ipld/go-ipld-prime](https://github.com/ipld/go-ipld-prime) from 0.9.0 to 0.19.0. - [Release notes](https://github.com/ipld/go-ipld-prime/releases) - [Changelog](https://github.com/ipld/go-ipld-prime/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipld/go-ipld-prime/compare/v0.9.0...v0.19.0) --- updated-dependencies: - dependency-name: github.com/ipld/go-ipld-prime dependency-type: indirect ... * Bump golang.org/x/net from 0.3.0 to 0.7.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.3.0 to 0.7.0. - [Release notes](https://github.com/golang/net/releases) - [Commits](https://github.com/golang/net/compare/v0.3.0...v0.7.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... * Small fixes. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * activate epoch * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review "--port flag". * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * Commented golangci. * staged stream sync v1.0 * fix protocol tests * fix spell * remove unused struct * fix rosetta test * add comments and refactor verify sig * add comments, remove extra function * add comment * refactor errors, rename metrics * refactor p2p host creation * fix initsync and host creation * fix short range hash chain * fix beacon node detection for p2p protocol * refactor stream peer cooldown and fix protocol beacon node field * refactor p2p host and routing * fix p2p discovery test issue * add MaxAdvertiseWaitTime to handle advertisements interval and address stream connection issue * terminal print the peer id and proto id * fix boot complete message when node is shut down * add new config option ( ForceReachabilityPublic ) to fix local-net consensus issue * fix self query issue * fix test NewDNSSyncingPeerProvider * [testnet] disable leader rotation * fix discovery issue for legacy sync * add watermark low/high options for p2p connection manager * add test for new conn manager flags * fix dedent * add comment to inform about p2p connection manager options * fix max height issue * add a separate log for get max height error * fix log * feat: triesInMemory flag * fix: panic if TriesInMemory is 1 to 2 * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed timers locks. * Removed fbft locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional logs. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * Fix formatting. * Rebased onto dev. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Fix review comment. * Go mod tidy. * Set to EpochTBD. * Fix tests. * [core] fix state handling of self destruct If a contract self destructs to self and then receives funds within the same transaction, it is possible for its stale state to be saved. This change removes that possibility by checking for deleted state objects before returning them. * Fixed race error. * rpc: add configurable http and `eth_call` timeout * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fixed code review. * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * staged stream sync v1.0 * refactor errors, rename metrics * fix p2p discovery test issue * add watermark low/high options for p2p connection manager * fix dedent * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Go mod tidy. * remove default timeouts * Rotate external leaders on non-beacon chains. * Fix nil panic. * Fixes. * Update singleton.go * evm: don't return extcode for validators Due to technical debt, validator information is stored in the code field of the address. The code field can be accessed in Solidity for an arbitrary address using `extcodesize`, `extcodehash`, and `extcodecopy` or helper commands (such as `address.code.Length`). The presence of this field is used by contract developers to (erroneously) deny smart contract access to other smart contracts (and therefore, validators). This PR fixes that oversight by returning the same values as other EOAs for known validator addresses. Obviously, it needs a hard fork that will be scheduled separately. * Fix context passing. * Clean up code. * Removed engine dependency. * Fix possible panic. * Clean up code. * Network type. * Fix tests. * Revert "Removed engine dependency." (#4392) * Revert "Fix tests." This reverts commit 597ba2d6f1ed54ff599b9d9b8940c7285ab1277a. * Revert "Network type." This reverts commit 5e1878aedca3989dc0f34161dae1833e43ca6a76. * Revert "Clean up code." This reverts commit 15885f4c9b9263746827172b4f4f56d0926d18e2. * Revert "Fix possible panic." This reverts commit 1a70d5eb66cdbf8a23791806b71a323eed320085. * Revert "Removed engine dependency." This reverts commit 8c2ff803f709f944cfc8b1278f35cf5b2cacf859. * gitignore the cache folder (#4389) * stable localnet with external validator (#4388) * stable localnet with external validator * ignore deploy config file comments * reduce node launched in localnet * update makefile * localnet configuration - add more fn * fix validator information command typo --------- Signed-off-by: dependabot[bot] Co-authored-by: Casey Gardiner <117784577+ONECasey@users.noreply.github.com> Co-authored-by: frozen <355847+Frozen@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Soph <35721420+sophoah@users.noreply.github.com> * chore: merge `main` into `dev` (#4415) * Mainnet Release Candidate 2023.1.2 (#4376) * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * Bump github.com/aws/aws-sdk-go from 1.33.0 to 1.34.0 Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.33.0 to 1.34.0. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/v1.34.0/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.33.0...v1.34.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Bump github.com/ipld/go-ipld-prime from 0.9.0 to 0.19.0 Bumps [github.com/ipld/go-ipld-prime](https://github.com/ipld/go-ipld-prime) from 0.9.0 to 0.19.0. - [Release notes](https://github.com/ipld/go-ipld-prime/releases) - [Changelog](https://github.com/ipld/go-ipld-prime/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipld/go-ipld-prime/compare/v0.9.0...v0.19.0) --- updated-dependencies: - dependency-name: github.com/ipld/go-ipld-prime dependency-type: indirect ... Signed-off-by: dependabot[bot] * Bump golang.org/x/net from 0.3.0 to 0.7.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.3.0 to 0.7.0. - [Release notes](https://github.com/golang/net/releases) - [Commits](https://github.com/golang/net/compare/v0.3.0...v0.7.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] * Small fixes. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * activate epoch * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review "--port flag". * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * Commented golangci. * staged stream sync v1.0 * fix protocol tests * fix spell * remove unused struct * fix rosetta test * add comments and refactor verify sig * add comments, remove extra function * add comment * refactor errors, rename metrics * refactor p2p host creation * fix initsync and host creation * fix short range hash chain * fix beacon node detection for p2p protocol * refactor stream peer cooldown and fix protocol beacon node field * refactor p2p host and routing * fix p2p discovery test issue * add MaxAdvertiseWaitTime to handle advertisements interval and address stream connection issue * terminal print the peer id and proto id * fix boot complete message when node is shut down * add new config option ( ForceReachabilityPublic ) to fix local-net consensus issue * fix self query issue * fix test NewDNSSyncingPeerProvider * [testnet] disable leader rotation * fix discovery issue for legacy sync * add watermark low/high options for p2p connection manager * add test for new conn manager flags * fix dedent * add comment to inform about p2p connection manager options * fix max height issue * add a separate log for get max height error * fix log * feat: triesInMemory flag * fix: panic if TriesInMemory is 1 to 2 * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed timers locks. * Removed fbft locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional logs. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * Fix formatting. * Rebased onto dev. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Fix review comment. * Go mod tidy. * Set to EpochTBD. * Fix tests. * [core] fix state handling of self destruct If a contract self destructs to self and then receives funds within the same transaction, it is possible for its stale state to be saved. This change removes that possibility by checking for deleted state objects before returning them. * Fixed race error. * rpc: add configurable http and `eth_call` timeout * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fixed code review. * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * staged stream sync v1.0 * refactor errors, rename metrics * fix p2p discovery test issue * add watermark low/high options for p2p connection manager * fix dedent * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Go mod tidy. * remove default timeouts * Rotate external leaders on non-beacon chains. * Fix nil panic. * Fixes. * Update singleton.go * evm: don't return extcode for validators Due to technical debt, validator information is stored in the code field of the address. The code field can be accessed in Solidity for an arbitrary address using `extcodesize`, `extcodehash`, and `extcodecopy` or helper commands (such as `address.code.Length`). The presence of this field is used by contract developers to (erroneously) deny smart contract access to other smart contracts (and therefore, validators). This PR fixes that oversight by returning the same values as other EOAs for known validator addresses. Obviously, it needs a hard fork that will be scheduled separately. * Fix context passing. * Clean up code. * Removed engine dependency. * Fix possible panic. * Clean up code. * Network type. * Fix tests. * Revert "Removed engine dependency." (#4392) * Revert "Fix tests." This reverts commit 597ba2d6f1ed54ff599b9d9b8940c7285ab1277a. * Revert "Network type." This reverts commit 5e1878aedca3989dc0f34161dae1833e43ca6a76. * Revert "Clean up code." This reverts commit 15885f4c9b9263746827172b4f4f56d0926d18e2. * Revert "Fix possible panic." This reverts commit 1a70d5eb66cdbf8a23791806b71a323eed320085. * Revert "Removed engine dependency." This reverts commit 8c2ff803f709f944cfc8b1278f35cf5b2cacf859. * gitignore the cache folder (#4389) * stable localnet with external validator (#4388) * stable localnet with external validator * ignore deploy config file comments * reduce node launched in localnet * update makefile * localnet configuration - add more fn * fix validator information command typo --------- Signed-off-by: dependabot[bot] Co-authored-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Co-authored-by: frozen <355847+Frozen@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Soph <35721420+sophoah@users.noreply.github.com> * build: update pinned curl version (#4394) Per the Alpine Linux package repositories, the version for cURL included with v3.16 has changed to revision 6 * consensus: replace type assert with test (#4398) If `consensus.finalityCounter` does not have anything stored (for example in Syncing mode), the `Load()` returns an interface that cannot be automatically asserted to an `int64`. This results in the node crashing. This commit fixes that. * Turn pprof default on with local saved files (#3894) * Turn pprof default on with local saved files * [pprof] change interval from 600s to 3600s * Revert "Turn pprof default on with local saved files (#3894)" (#4400) This reverts commit 78d26d7910a58ded3bfe689b3de07ea28d95d7d5. --------- Signed-off-by: dependabot[bot] Co-authored-by: Casey Gardiner <117784577+ONECasey@users.noreply.github.com> Co-authored-by: frozen <355847+Frozen@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Soph <35721420+sophoah@users.noreply.github.com> Co-authored-by: Jacky Wang * internal/params: set validator code fix hard forks (#4413) * internal/params: schedule hard forks Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> * internal/params: set localnet fee collect epoch 2 Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> --------- Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> * internal/params: schedule HIP28v2 + val code fix (#4416) Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> * Dev fix conflicts. (#4417) * Mainnet Release Candidate 2023.1.2 (#4376) * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * Bump github.com/aws/aws-sdk-go from 1.33.0 to 1.34.0 Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.33.0 to 1.34.0. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/v1.34.0/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.33.0...v1.34.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Bump github.com/ipld/go-ipld-prime from 0.9.0 to 0.19.0 Bumps [github.com/ipld/go-ipld-prime](https://github.com/ipld/go-ipld-prime) from 0.9.0 to 0.19.0. - [Release notes](https://github.com/ipld/go-ipld-prime/releases) - [Changelog](https://github.com/ipld/go-ipld-prime/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipld/go-ipld-prime/compare/v0.9.0...v0.19.0) --- updated-dependencies: - dependency-name: github.com/ipld/go-ipld-prime dependency-type: indirect ... Signed-off-by: dependabot[bot] * Bump golang.org/x/net from 0.3.0 to 0.7.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.3.0 to 0.7.0. - [Release notes](https://github.com/golang/net/releases) - [Commits](https://github.com/golang/net/compare/v0.3.0...v0.7.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] * Small fixes. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * activate epoch * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review "--port flag". * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * Commented golangci. * staged stream sync v1.0 * fix protocol tests * fix spell * remove unused struct * fix rosetta test * add comments and refactor verify sig * add comments, remove extra function * add comment * refactor errors, rename metrics * refactor p2p host creation * fix initsync and host creation * fix short range hash chain * fix beacon node detection for p2p protocol * refactor stream peer cooldown and fix protocol beacon node field * refactor p2p host and routing * fix p2p discovery test issue * add MaxAdvertiseWaitTime to handle advertisements interval and address stream connection issue * terminal print the peer id and proto id * fix boot complete message when node is shut down * add new config option ( ForceReachabilityPublic ) to fix local-net consensus issue * fix self query issue * fix test NewDNSSyncingPeerProvider * [testnet] disable leader rotation * fix discovery issue for legacy sync * add watermark low/high options for p2p connection manager * add test for new conn manager flags * fix dedent * add comment to inform about p2p connection manager options * fix max height issue * add a separate log for get max height error * fix log * feat: triesInMemory flag * fix: panic if TriesInMemory is 1 to 2 * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed timers locks. * Removed fbft locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional logs. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * Fix formatting. * Rebased onto dev. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Fix review comment. * Go mod tidy. * Set to EpochTBD. * Fix tests. * [core] fix state handling of self destruct If a contract self destructs to self and then receives funds within the same transaction, it is possible for its stale state to be saved. This change removes that possibility by checking for deleted state objects before returning them. * Fixed race error. * rpc: add configurable http and `eth_call` timeout * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fixed code review. * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * staged stream sync v1.0 * refactor errors, rename metrics * fix p2p discovery test issue * add watermark low/high options for p2p connection manager * fix dedent * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Go mod tidy. * remove default timeouts * Rotate external leaders on non-beacon chains. * Fix nil panic. * Fixes. * Update singleton.go * evm: don't return extcode for validators Due to technical debt, validator information is stored in the code field of the address. The code field can be accessed in Solidity for an arbitrary address using `extcodesize`, `extcodehash`, and `extcodecopy` or helper commands (such as `address.code.Length`). The presence of this field is used by contract developers to (erroneously) deny smart contract access to other smart contracts (and therefore, validators). This PR fixes that oversight by returning the same values as other EOAs for known validator addresses. Obviously, it needs a hard fork that will be scheduled separately. * Fix context passing. * Clean up code. * Removed engine dependency. * Fix possible panic. * Clean up code. * Network type. * Fix tests. * Revert "Removed engine dependency." (#4392) * Revert "Fix tests." This reverts commit 597ba2d6f1ed54ff599b9d9b8940c7285ab1277a. * Revert "Network type." This reverts commit 5e1878aedca3989dc0f34161dae1833e43ca6a76. * Revert "Clean up code." This reverts commit 15885f4c9b9263746827172b4f4f56d0926d18e2. * Revert "Fix possible panic." This reverts commit 1a70d5eb66cdbf8a23791806b71a323eed320085. * Revert "Removed engine dependency." This reverts commit 8c2ff803f709f944cfc8b1278f35cf5b2cacf859. * gitignore the cache folder (#4389) * stable localnet with external validator (#4388) * stable localnet with external validator * ignore deploy config file comments * reduce node launched in localnet * update makefile * localnet configuration - add more fn * fix validator information command typo --------- Signed-off-by: dependabot[bot] Co-authored-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Co-authored-by: frozen <355847+Frozen@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Soph <35721420+sophoah@users.noreply.github.com> * build: update pinned curl version (#4394) Per the Alpine Linux package repositories, the version for cURL included with v3.16 has changed to revision 6 * consensus: replace type assert with test (#4398) If `consensus.finalityCounter` does not have anything stored (for example in Syncing mode), the `Load()` returns an interface that cannot be automatically asserted to an `int64`. This results in the node crashing. This commit fixes that. * Turn pprof default on with local saved files (#3894) * Turn pprof default on with local saved files * [pprof] change interval from 600s to 3600s * Revert "Turn pprof default on with local saved files (#3894)" (#4400) This reverts commit 78d26d7910a58ded3bfe689b3de07ea28d95d7d5. * go mod tidy. * Increased wait time. --------- Signed-off-by: dependabot[bot] Co-authored-by: Casey Gardiner <117784577+ONECasey@users.noreply.github.com> Co-authored-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Soph <35721420+sophoah@users.noreply.github.com> Co-authored-by: Jacky Wang --------- Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Signed-off-by: dependabot[bot] Co-authored-by: frozen <355847+Frozen@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Soph <35721420+sophoah@users.noreply.github.com> Co-authored-by: Konstantin Co-authored-by: Gheis Mohammadi Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jacky Wang * internal/params: un-schedule mainnet hard forks (#4420) * internal/params: un-schedule mainnet hard forks and add logs to help debug the recent consensus loss * core/state: update logs * [hotfix] fix code hash conflicts (#4431) * fix code hash issue * goimports --------- Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Signed-off-by: dependabot[bot] Co-authored-by: Casey Gardiner <117784577+ONECasey@users.noreply.github.com> Co-authored-by: frozen <355847+Frozen@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Konstantin Co-authored-by: Gheis Mohammadi Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jacky Wang --- core/state/dump.go | 2 +- core/state/prefeth.go | 2 +- core/state/state_object.go | 108 +++++++++++++++---------- core/state/state_test.go | 2 +- core/state/statedb.go | 16 ++-- core/state/statedb_test.go | 22 ++--- core/vm/contracts_write.go | 2 +- core/vm/evm.go | 8 +- core/vm/instructions.go | 4 +- core/vm/interface.go | 4 +- hmy/tracers/block_tracer.go | 2 +- hmy/tracers/tracer.go | 2 +- internal/params/config.go | 4 +- rosetta/services/block.go | 4 +- rosetta/services/construction_check.go | 2 +- rpc/contract.go | 2 +- 16 files changed, 104 insertions(+), 82 deletions(-) diff --git a/core/state/dump.go b/core/state/dump.go index c4d70f6e00..d9e031ed4a 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -165,7 +165,7 @@ func (s *DB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []byte) addr := common.BytesToAddress(addrBytes) obj := newObject(s, addr, data) if !conf.SkipCode { - account.Code = obj.Code(s.db, false) + account.Code = obj.Code(s.db) } if !conf.SkipStorage { account.Storage = make(map[common.Hash]string) diff --git a/core/state/prefeth.go b/core/state/prefeth.go index 0b19f80d74..4d3b317bb1 100644 --- a/core/state/prefeth.go +++ b/core/state/prefeth.go @@ -100,7 +100,7 @@ func (s *DB) prefetchWorker(job *prefetchJob, jobs chan *prefetchJob) { addr := common.BytesToAddress(addrBytes) obj := newObject(s, addr, data) if data.CodeHash != nil { - obj.Code(s.db, false) + obj.Code(s.db) } // build account trie tree diff --git a/core/state/state_object.go b/core/state/state_object.go index 262f5136ee..20540371a9 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -32,7 +32,13 @@ import ( "github.com/harmony-one/harmony/staking" ) -var emptyCodeHash = crypto.Keccak256(nil) +var ( + // EmptyRootHash is the known root hash of an empty trie. + EmptyRootHash = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + + // EmptyCodeHash is the known hash of the empty EVM bytecode. + EmptyCodeHash = crypto.Keccak256Hash(nil) // c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470 +) // Code ... type Code []byte @@ -101,7 +107,7 @@ type Object struct { // empty returns whether the account is considered empty. func (s *Object) empty() bool { - return s.data.Nonce == 0 && s.data.Balance.Sign() == 0 && bytes.Equal(s.data.CodeHash, emptyCodeHash) + return s.data.Nonce == 0 && s.data.Balance.Sign() == 0 && bytes.Equal(s.data.CodeHash, EmptyCodeHash.Bytes()) } // Account is the Ethereum consensus representation of accounts. @@ -119,10 +125,10 @@ func newObject(db *DB, address common.Address, data types.StateAccount) *Object data.Balance = new(big.Int) } if data.CodeHash == nil { - data.CodeHash = types.EmptyCodeHash.Bytes() + data.CodeHash = EmptyCodeHash.Bytes() } if data.Root == (common.Hash{}) { - data.Root = types.EmptyRootHash + data.Root = EmptyRootHash } return &Object{ db: db, @@ -169,7 +175,7 @@ func (s *Object) getTrie(db Database) (Trie, error) { if s.trie == nil { // Try fetching from prefetcher first // We don't prefetch empty tries - if s.data.Root != types.EmptyRootHash && s.db.prefetcher != nil { + if s.data.Root != EmptyRootHash && s.db.prefetcher != nil { // When the miner is creating the pending state, there is no // prefetcher s.trie = s.db.prefetcher.trie(s.addrHash, s.data.Root) @@ -316,7 +322,7 @@ func (s *Object) finalise(prefetch bool) { slotsToPrefetch = append(slotsToPrefetch, common.CopyBytes(key[:])) // Copy needed for closure } } - if s.db.prefetcher != nil && prefetch && len(slotsToPrefetch) > 0 && s.data.Root != types.EmptyRootHash { + if s.db.prefetcher != nil && prefetch && len(slotsToPrefetch) > 0 && s.data.Root != EmptyRootHash { s.db.prefetcher.prefetch(s.addrHash, s.data.Root, slotsToPrefetch) } if len(s.dirtyStorage) > 0 { @@ -475,18 +481,18 @@ func (s *Object) setBalance(amount *big.Int) { func (s *Object) ReturnGas(gas *big.Int) {} func (s *Object) deepCopy(db *DB) *Object { - Object := newObject(db, s.address, s.data) + stateObject := newObject(db, s.address, s.data) if s.trie != nil { - Object.trie = db.db.CopyTrie(s.trie) + stateObject.trie = db.db.CopyTrie(s.trie) } - Object.code = s.code - Object.dirtyStorage = s.dirtyStorage.Copy() - Object.originStorage = s.originStorage.Copy() - Object.pendingStorage = s.pendingStorage.Copy() - Object.suicided = s.suicided - Object.dirtyCode = s.dirtyCode - Object.deleted = s.deleted - return Object + stateObject.code = s.code + stateObject.dirtyStorage = s.dirtyStorage.Copy() + stateObject.originStorage = s.originStorage.Copy() + stateObject.pendingStorage = s.pendingStorage.Copy() + stateObject.suicided = s.suicided + stateObject.dirtyCode = s.dirtyCode + stateObject.deleted = s.deleted + return stateObject } // @@ -499,27 +505,35 @@ func (s *Object) Address() common.Address { } // Code returns the contract/validator code associated with this object, if any. -func (s *Object) Code(db Database, isValidatorCode bool) []byte { +func (s *Object) Code(db Database) []byte { if s.code != nil { return s.code } - if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) { + if bytes.Equal(s.CodeHash(), EmptyCodeHash.Bytes()) { return nil } - if s.validatorWrapper || isValidatorCode { - code, err := db.ValidatorCode(s.addrHash, common.BytesToHash(s.CodeHash())) - if err != nil { - s.setError(fmt.Errorf("can't load validator code hash %x: %v", s.CodeHash(), err)) + var err error + code := []byte{} + // if it's not set for validator wrapper, then it may be either contract code or validator wrapper (old version of db + // don't have any prefix to differentiate between them) + // so, if it's not set for validator wrapper, we need to check contract code as well + if !s.validatorWrapper { + code, err = db.ContractCode(s.addrHash, common.BytesToHash(s.CodeHash())) + } + // if it couldn't load contract code or it is set to validator wrapper, then it tries to fetch validator wrapper code + if s.validatorWrapper || err != nil { + vCode, errVCode := db.ValidatorCode(s.addrHash, common.BytesToHash(s.CodeHash())) + if errVCode == nil && vCode != nil { + s.code = vCode + return vCode } - if code != nil { - s.code = code - return code + if s.validatorWrapper { + s.setError(fmt.Errorf("can't load validator code hash %x for account address hash %x : %v", s.CodeHash(), s.addrHash, err)) + } else { + s.setError(fmt.Errorf("can't load contract/validator code hash %x for account address hash %x : contract code error: %v, validator code error: %v", + s.CodeHash(), s.addrHash, err, errVCode)) } } - code, err := db.ContractCode(s.addrHash, common.BytesToHash(s.CodeHash())) - if err != nil { - s.setError(fmt.Errorf("can't load code hash %x: %v", s.CodeHash(), err)) - } s.code = code return code } @@ -527,31 +541,41 @@ func (s *Object) Code(db Database, isValidatorCode bool) []byte { // CodeSize returns the size of the contract/validator code associated with this object, // or zero if none. This method is an almost mirror of Code, but uses a cache // inside the database to avoid loading codes seen recently. -func (s *Object) CodeSize(db Database, isValidatorCode bool) int { +func (s *Object) CodeSize(db Database) int { if s.code != nil { return len(s.code) } - if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) { + if bytes.Equal(s.CodeHash(), EmptyCodeHash.Bytes()) { return 0 } - if s.validatorWrapper || isValidatorCode { - size, err := db.ValidatorCodeSize(s.addrHash, common.BytesToHash(s.CodeHash())) - if err != nil { - s.setError(fmt.Errorf("can't load validator code size %x: %v", s.CodeHash(), err)) - } - if size > 0 { + var err error + size := int(0) + + // if it's not set for validator wrapper, then it may be either contract code or validator wrapper (old version of db + // don't have any prefix to differentiate between them) + // so, if it's not set for validator wrapper, we need to check contract code as well + if !s.validatorWrapper { + size, err = db.ContractCodeSize(s.addrHash, common.BytesToHash(s.CodeHash())) + } + // if it couldn't get contract code or it is set to validator wrapper, then it tries to retrieve validator wrapper code + if s.validatorWrapper || err != nil { + vcSize, errVCSize := db.ValidatorCodeSize(s.addrHash, common.BytesToHash(s.CodeHash())) + if errVCSize == nil && vcSize > 0 { return size } - } - size, err := db.ContractCodeSize(s.addrHash, common.BytesToHash(s.CodeHash())) - if err != nil { - s.setError(fmt.Errorf("can't load code size %x: %v", s.CodeHash(), err)) + if s.validatorWrapper { + s.setError(fmt.Errorf("can't load validator code size %x for account address hash %x : %v", s.CodeHash(), s.addrHash, err)) + } else { + s.setError(fmt.Errorf("can't load contract/validator code size %x for account address hash %x : contract code size error: %v, validator code size error: %v", + s.CodeHash(), s.addrHash, err, errVCSize)) + } + s.setError(fmt.Errorf("can't load code size %x (validator wrapper: %t): %v", s.CodeHash(), s.validatorWrapper, err)) } return size } func (s *Object) SetCode(codeHash common.Hash, code []byte, isValidatorCode bool) { - prevcode := s.Code(s.db.db, isValidatorCode) + prevcode := s.Code(s.db.db) s.db.journal.append(codeChange{ account: &s.address, prevhash: s.CodeHash(), diff --git a/core/state/state_test.go b/core/state/state_test.go index 9d209a80be..52e9a6e6ec 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -194,7 +194,7 @@ func TestSnapshot2(t *testing.T) { so0Restored := state.getStateObject(stateobjaddr0) // Update lazily-loaded values before comparing. so0Restored.GetState(state.db, storageaddr) - so0Restored.Code(state.db, false) + so0Restored.Code(state.db) // non-deleted is equal (restored) compareStateObjects(so0Restored, so0, t) diff --git a/core/state/statedb.go b/core/state/statedb.go index 30692e4d44..9c1eefde82 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -342,18 +342,18 @@ func (db *DB) BlockHash() common.Hash { return db.bhash } -func (db *DB) GetCode(addr common.Address, isValidatorCode bool) []byte { +func (db *DB) GetCode(addr common.Address) []byte { Object := db.getStateObject(addr) if Object != nil { - return Object.Code(db.db, isValidatorCode) + return Object.Code(db.db) } return nil } -func (db *DB) GetCodeSize(addr common.Address, isValidatorCode bool) int { +func (db *DB) GetCodeSize(addr common.Address) int { Object := db.getStateObject(addr) if Object != nil { - return Object.CodeSize(db.db, isValidatorCode) + return Object.CodeSize(db.db) } return 0 } @@ -1241,13 +1241,11 @@ func (db *DB) ValidatorWrapper( return copyValidatorWrapperIfNeeded(cached, sendOriginal, copyDelegations), nil } - by := db.GetCode(addr, true) + by := db.GetCode(addr) if len(by) == 0 { - by = db.GetCode(addr, false) - if len(by) == 0 { - return nil, ErrAddressNotPresent - } + return nil, ErrAddressNotPresent } + val := stk.ValidatorWrapper{} if err := rlp.DecodeBytes(by, &val); err != nil { return nil, errors.Wrapf( diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index f4277809bd..538edac160 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -452,9 +452,9 @@ func (test *snapshotTest) checkEqual(state, checkstate *DB) error { checkeq("HasSuicided", state.HasSuicided(addr), checkstate.HasSuicided(addr)) checkeq("GetBalance", state.GetBalance(addr), checkstate.GetBalance(addr)) checkeq("GetNonce", state.GetNonce(addr), checkstate.GetNonce(addr)) - checkeq("GetCode", state.GetCode(addr, false), checkstate.GetCode(addr, false)) + checkeq("GetCode", state.GetCode(addr), checkstate.GetCode(addr)) checkeq("GetCodeHash", state.GetCodeHash(addr), checkstate.GetCodeHash(addr)) - checkeq("GetCodeSize", state.GetCodeSize(addr, false), checkstate.GetCodeSize(addr, false)) + checkeq("GetCodeSize", state.GetCodeSize(addr), checkstate.GetCodeSize(addr)) // Check storage. if obj := state.getStateObject(addr); obj != nil { state.ForEachStorage(addr, func(key, value common.Hash) bool { @@ -532,7 +532,7 @@ func TestCopyCommitCopy(t *testing.T) { if balance := state.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("initial balance mismatch: have %v, want %v", balance, 42) } - if code := state.GetCode(addr, false); !bytes.Equal(code, []byte("hello")) { + if code := state.GetCode(addr); !bytes.Equal(code, []byte("hello")) { t.Fatalf("initial code mismatch: have %x, want %x", code, []byte("hello")) } if val := state.GetState(addr, skey); val != sval { @@ -546,7 +546,7 @@ func TestCopyCommitCopy(t *testing.T) { if balance := copyOne.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("first copy pre-commit balance mismatch: have %v, want %v", balance, 42) } - if code := copyOne.GetCode(addr, false); !bytes.Equal(code, []byte("hello")) { + if code := copyOne.GetCode(addr); !bytes.Equal(code, []byte("hello")) { t.Fatalf("first copy pre-commit code mismatch: have %x, want %x", code, []byte("hello")) } if val := copyOne.GetState(addr, skey); val != sval { @@ -560,7 +560,7 @@ func TestCopyCommitCopy(t *testing.T) { if balance := copyOne.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("first copy post-commit balance mismatch: have %v, want %v", balance, 42) } - if code := copyOne.GetCode(addr, false); !bytes.Equal(code, []byte("hello")) { + if code := copyOne.GetCode(addr); !bytes.Equal(code, []byte("hello")) { t.Fatalf("first copy post-commit code mismatch: have %x, want %x", code, []byte("hello")) } if val := copyOne.GetState(addr, skey); val != sval { @@ -574,7 +574,7 @@ func TestCopyCommitCopy(t *testing.T) { if balance := copyTwo.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("second copy balance mismatch: have %v, want %v", balance, 42) } - if code := copyTwo.GetCode(addr, false); !bytes.Equal(code, []byte("hello")) { + if code := copyTwo.GetCode(addr); !bytes.Equal(code, []byte("hello")) { t.Fatalf("second copy code mismatch: have %x, want %x", code, []byte("hello")) } if val := copyTwo.GetState(addr, skey); val != sval { @@ -604,7 +604,7 @@ func TestCopyCopyCommitCopy(t *testing.T) { if balance := state.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("initial balance mismatch: have %v, want %v", balance, 42) } - if code := state.GetCode(addr, false); !bytes.Equal(code, []byte("hello")) { + if code := state.GetCode(addr); !bytes.Equal(code, []byte("hello")) { t.Fatalf("initial code mismatch: have %x, want %x", code, []byte("hello")) } if val := state.GetState(addr, skey); val != sval { @@ -618,7 +618,7 @@ func TestCopyCopyCommitCopy(t *testing.T) { if balance := copyOne.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("first copy balance mismatch: have %v, want %v", balance, 42) } - if code := copyOne.GetCode(addr, false); !bytes.Equal(code, []byte("hello")) { + if code := copyOne.GetCode(addr); !bytes.Equal(code, []byte("hello")) { t.Fatalf("first copy code mismatch: have %x, want %x", code, []byte("hello")) } if val := copyOne.GetState(addr, skey); val != sval { @@ -632,7 +632,7 @@ func TestCopyCopyCommitCopy(t *testing.T) { if balance := copyTwo.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("second copy pre-commit balance mismatch: have %v, want %v", balance, 42) } - if code := copyTwo.GetCode(addr, false); !bytes.Equal(code, []byte("hello")) { + if code := copyTwo.GetCode(addr); !bytes.Equal(code, []byte("hello")) { t.Fatalf("second copy pre-commit code mismatch: have %x, want %x", code, []byte("hello")) } if val := copyTwo.GetState(addr, skey); val != sval { @@ -645,7 +645,7 @@ func TestCopyCopyCommitCopy(t *testing.T) { if balance := copyTwo.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("second copy post-commit balance mismatch: have %v, want %v", balance, 42) } - if code := copyTwo.GetCode(addr, false); !bytes.Equal(code, []byte("hello")) { + if code := copyTwo.GetCode(addr); !bytes.Equal(code, []byte("hello")) { t.Fatalf("second copy post-commit code mismatch: have %x, want %x", code, []byte("hello")) } if val := copyTwo.GetState(addr, skey); val != sval { @@ -659,7 +659,7 @@ func TestCopyCopyCommitCopy(t *testing.T) { if balance := copyThree.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("third copy balance mismatch: have %v, want %v", balance, 42) } - if code := copyThree.GetCode(addr, false); !bytes.Equal(code, []byte("hello")) { + if code := copyThree.GetCode(addr); !bytes.Equal(code, []byte("hello")) { t.Fatalf("third copy code mismatch: have %x, want %x", code, []byte("hello")) } if val := copyThree.GetState(addr, skey); val != sval { diff --git a/core/vm/contracts_write.go b/core/vm/contracts_write.go index 7e24eb9a27..46ba7fe923 100644 --- a/core/vm/contracts_write.go +++ b/core/vm/contracts_write.go @@ -242,7 +242,7 @@ func (c *crossShardXferPrecompile) RunWriteCapable( return nil, err } // validate not a contract (toAddress can still be a contract) - if len(evm.StateDB.GetCode(fromAddress, false)) > 0 && !evm.IsValidator(evm.StateDB, fromAddress) { + if len(evm.StateDB.GetCode(fromAddress)) > 0 && !evm.IsValidator(evm.StateDB, fromAddress) { return nil, errors.New("cross shard xfer not yet implemented for contracts") } // can't have too many shards diff --git a/core/vm/evm.go b/core/vm/evm.go index 53da390dba..55f6f8c93d 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -336,7 +336,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas evm.Transfer(evm.StateDB, caller.Address(), to.Address(), value, txType) codeHash := evm.StateDB.GetCodeHash(addr) - code := evm.StateDB.GetCode(addr, false) + code := evm.StateDB.GetCode(addr) // If address is a validator address, then it's not a smart contract address // we don't use its code and codeHash fields if evm.Context.IsValidator(evm.StateDB, addr) { @@ -402,7 +402,7 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, // EVM. The contract is a scoped environment for this execution context // only. contract := NewContract(caller, to, value, gas) - contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr, false)) + contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) ret, err = run(evm, contract, input, false) if err != nil { @@ -435,7 +435,7 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by // Initialise a new contract and make initialise the delegate values contract := NewContract(caller, to, nil, gas).AsDelegate() - contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr, false)) + contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) ret, err = run(evm, contract, input, false) if err != nil { @@ -468,7 +468,7 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte // EVM. The contract is a scoped environment for this execution context // only. contract := NewContract(caller, to, new(big.Int), gas) - contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr, false)) + contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) // We do an AddBalance of zero here, just in order to trigger a touch. // This doesn't matter on Mainnet, where all empties are gone at the time of Byzantium, diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 73f826a63f..091ba28ff6 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -488,7 +488,7 @@ func opExtCodeSize(pc *uint64, interpreter *EVMInterpreter, contract *Contract, slot.SetUint64(0) return nil, nil } - slot.SetUint64(uint64(interpreter.evm.StateDB.GetCodeSize(common.BigToAddress(slot), false))) + slot.SetUint64(uint64(interpreter.evm.StateDB.GetCodeSize(common.BigToAddress(slot)))) return nil, nil } @@ -528,7 +528,7 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, contract *Contract, // for EOAs that are not validators, statedb returns nil code = nil } else { - code = interpreter.evm.StateDB.GetCode(addr, false) + code = interpreter.evm.StateDB.GetCode(addr) } codeCopy := getDataBig(code, codeOffset, length) memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy) diff --git a/core/vm/interface.go b/core/vm/interface.go index 3b481fd69e..0d600ca16f 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -38,9 +38,9 @@ type StateDB interface { SetNonce(common.Address, uint64) GetCodeHash(common.Address) common.Hash - GetCode(common.Address, bool) []byte + GetCode(common.Address) []byte SetCode(common.Address, []byte, bool) - GetCodeSize(common.Address, bool) int + GetCodeSize(common.Address) int ValidatorWrapper(common.Address, bool, bool) (*staking.ValidatorWrapper, error) UpdateValidatorWrapper(common.Address, *staking.ValidatorWrapper) error diff --git a/hmy/tracers/block_tracer.go b/hmy/tracers/block_tracer.go index daaf4171d0..12391c6fb9 100644 --- a/hmy/tracers/block_tracer.go +++ b/hmy/tracers/block_tracer.go @@ -353,7 +353,7 @@ func (jst *ParityBlockTracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, ret := stackPeek(0) if ret.Sign() != 0 { call.to = common.BigToAddress(ret) - call.output = env.StateDB.GetCode(call.to, false) + call.output = env.StateDB.GetCode(call.to) } else if call.err == nil { call.err = errors.New("internal failure") } diff --git a/hmy/tracers/tracer.go b/hmy/tracers/tracer.go index 69d52e3fdc..bc349a5147 100644 --- a/hmy/tracers/tracer.go +++ b/hmy/tracers/tracer.go @@ -210,7 +210,7 @@ func (dw *dbWrapper) pushObject(vm *duktape.Context) { // Push the wrapper for statedb.GetCode vm.PushGoFunction(func(ctx *duktape.Context) int { - code := dw.db.GetCode(common.BytesToAddress(popSlice(ctx)), false) + code := dw.db.GetCode(common.BytesToAddress(popSlice(ctx))) ptr := ctx.PushFixedBuffer(len(code)) copy(makeSlice(ptr, uint(len(code))), code) diff --git a/internal/params/config.go b/internal/params/config.go index 43e538243c..f9ccbaa64e 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -71,8 +71,8 @@ var ( AllowlistEpoch: EpochTBD, LeaderRotationExternalNonBeaconLeaders: EpochTBD, LeaderRotationExternalBeaconLeaders: EpochTBD, - FeeCollectEpoch: big.NewInt(1451), // 2023-05-17 04:02:00+00:00 - ValidatorCodeFixEpoch: big.NewInt(1451), + FeeCollectEpoch: EpochTBD, + ValidatorCodeFixEpoch: EpochTBD, } // TestnetChainConfig contains the chain parameters to run a node on the harmony test network. diff --git a/rosetta/services/block.go b/rosetta/services/block.go index 8a939b1081..c488c5ff95 100644 --- a/rosetta/services/block.go +++ b/rosetta/services/block.go @@ -178,11 +178,11 @@ func (s *BlockAPI) BlockTransaction( // check for contract related operations, if it is a plain transaction. if txInfo.tx.To() != nil { // possible call to existing contract so fetch relevant data - contractInfo.ContractCode = state.GetCode(*txInfo.tx.To(), false) + contractInfo.ContractCode = state.GetCode(*txInfo.tx.To()) contractInfo.ContractAddress = txInfo.tx.To() } else { // contract creation, so address is in receipt - contractInfo.ContractCode = state.GetCode(txInfo.receipt.ContractAddress, false) + contractInfo.ContractCode = state.GetCode(txInfo.receipt.ContractAddress) contractInfo.ContractAddress = &txInfo.receipt.ContractAddress } contractInfo.ExecutionResult, rosettaError = s.getTransactionTrace(ctx, blk, txInfo) diff --git a/rosetta/services/construction_check.go b/rosetta/services/construction_check.go index d08ed6e8f3..c842770ab7 100644 --- a/rosetta/services/construction_check.go +++ b/rosetta/services/construction_check.go @@ -289,7 +289,7 @@ func (s *ConstructAPI) ConstructionMetadata( GasPrice: sugNativePrice, GasLimit: estGasUsed, Transaction: options.TransactionMetadata, - ContractCode: state.GetCode(contractAddress, false), + ContractCode: state.GetCode(contractAddress), EvmErrorMessage: evmErrorMsg, EvmReturn: evmReturn, }) diff --git a/rpc/contract.go b/rpc/contract.go index daed35edd7..abcb4f9418 100644 --- a/rpc/contract.go +++ b/rpc/contract.go @@ -123,7 +123,7 @@ func (s *PublicContractService) GetCode( DoMetricRPCQueryInfo(GetCode, FailedNumber) return nil, err } - code := state.GetCode(address, false) + code := state.GetCode(address) // Response output is the same for all versions return code, state.Error() From a18e1e44bfedeb432a05f0c4ade3b7700284de79 Mon Sep 17 00:00:00 2001 From: Konstantin <355847+Frozen@users.noreply.github.com> Date: Fri, 19 May 2023 22:00:48 -0400 Subject: [PATCH 406/420] Feature: generating protoflles with docker. (#4427) * Gen proto files. * Clean up. * Fix. * Buildx multiplatform. * Added platform. * Fix generate. * Fix generate. * Fix generate. --- .travis.yml | 3 - Makefile | 3 + api/proto/message/gen.sh | 4 +- api/proto/message/message.go | 2 +- api/proto/message/message.pb.go | 82 +++++++++---------- api/proto/message/message_grpc.pb.go | 12 ++- .../legacysync/downloader/Proto.Dockerfile | 9 ++ api/service/legacysync/downloader/build.sh | 4 + api/service/legacysync/downloader/gen.sh | 5 -- .../legacysync/downloader/proto/downloader.go | 3 +- .../downloader/proto/downloader.pb.go | 4 +- .../downloader/proto/downloader_grpc.pb.go | 12 ++- .../legacysync/downloader/proto/gen.sh | 9 ++ p2p/stream/protocols/sync/message/gen.sh | 3 + p2p/stream/protocols/sync/message/generate.go | 2 +- p2p/stream/protocols/sync/message/msg.pb.go | 4 +- scripts/gogenerate.sh | 2 +- scripts/install_build_tools.sh | 15 ---- scripts/install_protoc.sh | 66 --------------- 19 files changed, 101 insertions(+), 143 deletions(-) create mode 100644 api/service/legacysync/downloader/Proto.Dockerfile create mode 100755 api/service/legacysync/downloader/build.sh delete mode 100755 api/service/legacysync/downloader/gen.sh create mode 100755 api/service/legacysync/downloader/proto/gen.sh create mode 100755 p2p/stream/protocols/sync/message/gen.sh delete mode 100755 scripts/install_build_tools.sh delete mode 100755 scripts/install_protoc.sh diff --git a/.travis.yml b/.travis.yml index 514f8e2fb4..cf99b47e61 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,13 +22,10 @@ install: - git clone https://github.com/harmony-one/harmony-test.git $GOPATH/src/github.com/harmony-one/harmony-test - (cd $GOPATH/src/github.com/harmony-one/mcl; make -j4) - (cd $GOPATH/src/github.com/harmony-one/bls; make BLS_SWAP_G=1 -j4) - - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26 - - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1 # - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.41.1 - make go-get - go install golang.org/x/tools/cmd/goimports@latest - go install github.com/harmony-ek/gencodec@latest - - scripts/install_build_tools.sh script: - ${TEST} after_success: diff --git a/Makefile b/Makefile index 052d6ad420..ee1d40faa0 100644 --- a/Makefile +++ b/Makefile @@ -164,3 +164,6 @@ go-test: docker: docker build --pull -t harmonyone/$(PKGNAME):latest -f scripts/docker/Dockerfile . + +travis_go_checker: + bash ./scripts/travis_go_checker.sh diff --git a/api/proto/message/gen.sh b/api/proto/message/gen.sh index 5dc2fc1bc8..bd667421e3 100755 --- a/api/proto/message/gen.sh +++ b/api/proto/message/gen.sh @@ -1 +1,3 @@ -protoc -I ./ message.proto --go_out=. --go-grpc_out=. +#!/bin/bash + +docker run --platform linux/amd64 -v ${PWD}:/tmp ${PROTOC_IMAGE} /tmp/message.proto diff --git a/api/proto/message/message.go b/api/proto/message/message.go index 8ba69bcf6a..22f91a10e7 100644 --- a/api/proto/message/message.go +++ b/api/proto/message/message.go @@ -1,3 +1,3 @@ package message -//go:generate protoc message.proto --go_out=. --go-grpc_out=. +//go:generate ./gen.sh diff --git a/api/proto/message/message.pb.go b/api/proto/message/message.pb.go index a72c0dd441..b1367f3d7d 100644 --- a/api/proto/message/message.pb.go +++ b/api/proto/message/message.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.26.0 -// protoc v3.15.8 +// protoc-gen-go v1.30.0 +// protoc v3.12.4 // source: message.proto package message @@ -25,9 +25,9 @@ type ServiceType int32 const ( ServiceType_CONSENSUS ServiceType = 0 - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. ServiceType_STAKING ServiceType = 1 - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. ServiceType_DRAND ServiceType = 2 ServiceType_CLIENT_SUPPORT ServiceType = 3 ) @@ -79,7 +79,7 @@ func (ServiceType) EnumDescriptor() ([]byte, []int) { type MessageType int32 const ( - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. MessageType_NEWNODE_BEACON_STAKING MessageType = 0 MessageType_ANNOUNCE MessageType = 1 MessageType_PREPARE MessageType = 2 @@ -88,11 +88,11 @@ const ( MessageType_COMMITTED MessageType = 5 MessageType_VIEWCHANGE MessageType = 6 MessageType_NEWVIEW MessageType = 7 - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. MessageType_DRAND_INIT MessageType = 10 - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. MessageType_DRAND_COMMIT MessageType = 11 - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. MessageType_LOTTERY_REQUEST MessageType = 12 // it should be either ENTER or GETPLAYERS but it will be removed later. ) @@ -156,11 +156,11 @@ func (MessageType) EnumDescriptor() ([]byte, []int) { type LotteryRequest_Type int32 const ( - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. LotteryRequest_ENTER LotteryRequest_Type = 0 - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. LotteryRequest_RESULT LotteryRequest_Type = 1 - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. LotteryRequest_PICK_WINNER LotteryRequest_Type = 2 ) @@ -288,7 +288,7 @@ func (m *Message) GetRequest() isMessage_Request { return nil } -// Deprecated: Do not use. +// Deprecated: Marked as deprecated in message.proto. func (x *Message) GetStaking() *StakingRequest { if x, ok := x.GetRequest().(*Message_Staking); ok { return x.Staking @@ -303,7 +303,7 @@ func (x *Message) GetConsensus() *ConsensusRequest { return nil } -// Deprecated: Do not use. +// Deprecated: Marked as deprecated in message.proto. func (x *Message) GetDrand() *DrandRequest { if x, ok := x.GetRequest().(*Message_Drand); ok { return x.Drand @@ -318,7 +318,7 @@ func (x *Message) GetViewchange() *ViewChangeRequest { return nil } -// Deprecated: Do not use. +// Deprecated: Marked as deprecated in message.proto. func (x *Message) GetLotteryRequest() *LotteryRequest { if x, ok := x.GetRequest().(*Message_LotteryRequest); ok { return x.LotteryRequest @@ -331,7 +331,7 @@ type isMessage_Request interface { } type Message_Staking struct { - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. Staking *StakingRequest `protobuf:"bytes,4,opt,name=staking,proto3,oneof"` } @@ -340,7 +340,7 @@ type Message_Consensus struct { } type Message_Drand struct { - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. Drand *DrandRequest `protobuf:"bytes,6,opt,name=drand,proto3,oneof"` } @@ -351,7 +351,7 @@ type Message_Viewchange struct { type Message_LotteryRequest struct { // Refactor this later after demo. // - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. LotteryRequest *LotteryRequest `protobuf:"bytes,8,opt,name=lottery_request,json=lotteryRequest,proto3,oneof"` } @@ -431,7 +431,7 @@ func (m *Response) GetResponse() isResponse_Response { return nil } -// Deprecated: Do not use. +// Deprecated: Marked as deprecated in message.proto. func (x *Response) GetLotteryResponse() *LotteryResponse { if x, ok := x.GetResponse().(*Response_LotteryResponse); ok { return x.LotteryResponse @@ -444,7 +444,7 @@ type isResponse_Response interface { } type Response_LotteryResponse struct { - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. LotteryResponse *LotteryResponse `protobuf:"bytes,3,opt,name=lottery_response,json=lotteryResponse,proto3,oneof"` } @@ -455,9 +455,9 @@ type LotteryResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. Players []string `protobuf:"bytes,2,rep,name=players,proto3" json:"players,omitempty"` - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. Balances []string `protobuf:"bytes,3,rep,name=balances,proto3" json:"balances,omitempty"` } @@ -493,7 +493,7 @@ func (*LotteryResponse) Descriptor() ([]byte, []int) { return file_message_proto_rawDescGZIP(), []int{2} } -// Deprecated: Do not use. +// Deprecated: Marked as deprecated in message.proto. func (x *LotteryResponse) GetPlayers() []string { if x != nil { return x.Players @@ -501,7 +501,7 @@ func (x *LotteryResponse) GetPlayers() []string { return nil } -// Deprecated: Do not use. +// Deprecated: Marked as deprecated in message.proto. func (x *LotteryResponse) GetBalances() []string { if x != nil { return x.Balances @@ -514,11 +514,11 @@ type LotteryRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. Type LotteryRequest_Type `protobuf:"varint,1,opt,name=type,proto3,enum=message.LotteryRequest_Type" json:"type,omitempty"` - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. PrivateKey string `protobuf:"bytes,2,opt,name=private_key,json=privateKey,proto3" json:"private_key,omitempty"` - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. Amount int64 `protobuf:"varint,3,opt,name=amount,proto3" json:"amount,omitempty"` } @@ -554,7 +554,7 @@ func (*LotteryRequest) Descriptor() ([]byte, []int) { return file_message_proto_rawDescGZIP(), []int{3} } -// Deprecated: Do not use. +// Deprecated: Marked as deprecated in message.proto. func (x *LotteryRequest) GetType() LotteryRequest_Type { if x != nil { return x.Type @@ -562,7 +562,7 @@ func (x *LotteryRequest) GetType() LotteryRequest_Type { return LotteryRequest_ENTER } -// Deprecated: Do not use. +// Deprecated: Marked as deprecated in message.proto. func (x *LotteryRequest) GetPrivateKey() string { if x != nil { return x.PrivateKey @@ -570,7 +570,7 @@ func (x *LotteryRequest) GetPrivateKey() string { return "" } -// Deprecated: Do not use. +// Deprecated: Marked as deprecated in message.proto. func (x *LotteryRequest) GetAmount() int64 { if x != nil { return x.Amount @@ -584,9 +584,9 @@ type StakingRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. Transaction []byte `protobuf:"bytes,1,opt,name=transaction,proto3" json:"transaction,omitempty"` - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. NodeId string `protobuf:"bytes,2,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` } @@ -622,7 +622,7 @@ func (*StakingRequest) Descriptor() ([]byte, []int) { return file_message_proto_rawDescGZIP(), []int{4} } -// Deprecated: Do not use. +// Deprecated: Marked as deprecated in message.proto. func (x *StakingRequest) GetTransaction() []byte { if x != nil { return x.Transaction @@ -630,7 +630,7 @@ func (x *StakingRequest) GetTransaction() []byte { return nil } -// Deprecated: Do not use. +// Deprecated: Marked as deprecated in message.proto. func (x *StakingRequest) GetNodeId() string { if x != nil { return x.NodeId @@ -746,13 +746,13 @@ type DrandRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. ShardId uint32 `protobuf:"varint,1,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"` - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. SenderPubkey []byte `protobuf:"bytes,2,opt,name=sender_pubkey,json=senderPubkey,proto3" json:"sender_pubkey,omitempty"` - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. BlockHash []byte `protobuf:"bytes,3,opt,name=block_hash,json=blockHash,proto3" json:"block_hash,omitempty"` - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in message.proto. Payload []byte `protobuf:"bytes,4,opt,name=payload,proto3" json:"payload,omitempty"` } @@ -788,7 +788,7 @@ func (*DrandRequest) Descriptor() ([]byte, []int) { return file_message_proto_rawDescGZIP(), []int{6} } -// Deprecated: Do not use. +// Deprecated: Marked as deprecated in message.proto. func (x *DrandRequest) GetShardId() uint32 { if x != nil { return x.ShardId @@ -796,7 +796,7 @@ func (x *DrandRequest) GetShardId() uint32 { return 0 } -// Deprecated: Do not use. +// Deprecated: Marked as deprecated in message.proto. func (x *DrandRequest) GetSenderPubkey() []byte { if x != nil { return x.SenderPubkey @@ -804,7 +804,7 @@ func (x *DrandRequest) GetSenderPubkey() []byte { return nil } -// Deprecated: Do not use. +// Deprecated: Marked as deprecated in message.proto. func (x *DrandRequest) GetBlockHash() []byte { if x != nil { return x.BlockHash @@ -812,7 +812,7 @@ func (x *DrandRequest) GetBlockHash() []byte { return nil } -// Deprecated: Do not use. +// Deprecated: Marked as deprecated in message.proto. func (x *DrandRequest) GetPayload() []byte { if x != nil { return x.Payload diff --git a/api/proto/message/message_grpc.pb.go b/api/proto/message/message_grpc.pb.go index e474fe0723..82a08b9b3d 100644 --- a/api/proto/message/message_grpc.pb.go +++ b/api/proto/message/message_grpc.pb.go @@ -1,4 +1,8 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v3.12.4 +// source: message.proto package message @@ -14,6 +18,10 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + ClientService_Process_FullMethodName = "/message.ClientService/Process" +) + // ClientServiceClient is the client API for ClientService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -31,7 +39,7 @@ func NewClientServiceClient(cc grpc.ClientConnInterface) ClientServiceClient { func (c *clientServiceClient) Process(ctx context.Context, in *Message, opts ...grpc.CallOption) (*Response, error) { out := new(Response) - err := c.cc.Invoke(ctx, "/message.ClientService/Process", in, out, opts...) + err := c.cc.Invoke(ctx, ClientService_Process_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -76,7 +84,7 @@ func _ClientService_Process_Handler(srv interface{}, ctx context.Context, dec fu } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/message.ClientService/Process", + FullMethod: ClientService_Process_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ClientServiceServer).Process(ctx, req.(*Message)) diff --git a/api/service/legacysync/downloader/Proto.Dockerfile b/api/service/legacysync/downloader/Proto.Dockerfile new file mode 100644 index 0000000000..5591abb476 --- /dev/null +++ b/api/service/legacysync/downloader/Proto.Dockerfile @@ -0,0 +1,9 @@ +FROM golang:1.19-bullseye + +RUN apt update +RUN apt install -y protobuf-compiler +RUN protoc --version +RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30.0 +RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3 + +ENTRYPOINT ["protoc", "-I=/tmp", "--go_out=/tmp", "--go-grpc_out=/tmp"] \ No newline at end of file diff --git a/api/service/legacysync/downloader/build.sh b/api/service/legacysync/downloader/build.sh new file mode 100755 index 0000000000..b0193985a9 --- /dev/null +++ b/api/service/legacysync/downloader/build.sh @@ -0,0 +1,4 @@ + +# if command fails, try this +# docker buildx create --use +docker buildx build -t frozen621/harmony-proto:latest --platform linux/amd64,linux/arm64 -f Proto.Dockerfile --progress=plain . \ No newline at end of file diff --git a/api/service/legacysync/downloader/gen.sh b/api/service/legacysync/downloader/gen.sh deleted file mode 100755 index 04c9286e71..0000000000 --- a/api/service/legacysync/downloader/gen.sh +++ /dev/null @@ -1,5 +0,0 @@ -# used versions -#go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26 -#go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1 -SRC_DIR=$(dirname $0) -protoc -I ${SRC_DIR}/proto/ ${SRC_DIR}/proto/downloader.proto --go_out=${SRC_DIR}/proto --go-grpc_out=${SRC_DIR}/proto diff --git a/api/service/legacysync/downloader/proto/downloader.go b/api/service/legacysync/downloader/proto/downloader.go index 023a87eb77..30fe9082d1 100644 --- a/api/service/legacysync/downloader/proto/downloader.go +++ b/api/service/legacysync/downloader/proto/downloader.go @@ -1,4 +1,5 @@ package downloader ///go:generate protoc downloader.proto --go_out=plugins=grpc:. -//go:generate protoc downloader.proto --go_out=. --go-grpc_out=. +///go:generate protoc downloader.proto --go_out=. --go-grpc_out=. +//go:generate ./gen.sh diff --git a/api/service/legacysync/downloader/proto/downloader.pb.go b/api/service/legacysync/downloader/proto/downloader.pb.go index d93550bf09..c0013dc226 100644 --- a/api/service/legacysync/downloader/proto/downloader.pb.go +++ b/api/service/legacysync/downloader/proto/downloader.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.26.0 -// protoc v3.15.8 +// protoc-gen-go v1.30.0 +// protoc v3.12.4 // source: downloader.proto package downloader diff --git a/api/service/legacysync/downloader/proto/downloader_grpc.pb.go b/api/service/legacysync/downloader/proto/downloader_grpc.pb.go index 1360c3eeec..cfe2f27f61 100644 --- a/api/service/legacysync/downloader/proto/downloader_grpc.pb.go +++ b/api/service/legacysync/downloader/proto/downloader_grpc.pb.go @@ -1,4 +1,8 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v3.12.4 +// source: downloader.proto package downloader @@ -14,6 +18,10 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + Downloader_Query_FullMethodName = "/downloader.Downloader/Query" +) + // DownloaderClient is the client API for Downloader service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -31,7 +39,7 @@ func NewDownloaderClient(cc grpc.ClientConnInterface) DownloaderClient { func (c *downloaderClient) Query(ctx context.Context, in *DownloaderRequest, opts ...grpc.CallOption) (*DownloaderResponse, error) { out := new(DownloaderResponse) - err := c.cc.Invoke(ctx, "/downloader.Downloader/Query", in, out, opts...) + err := c.cc.Invoke(ctx, Downloader_Query_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -76,7 +84,7 @@ func _Downloader_Query_Handler(srv interface{}, ctx context.Context, dec func(in } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/downloader.Downloader/Query", + FullMethod: Downloader_Query_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(DownloaderServer).Query(ctx, req.(*DownloaderRequest)) diff --git a/api/service/legacysync/downloader/proto/gen.sh b/api/service/legacysync/downloader/proto/gen.sh new file mode 100755 index 0000000000..23e55aefb4 --- /dev/null +++ b/api/service/legacysync/downloader/proto/gen.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# used versions +#go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26 +#go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1 +#SRC_DIR=$(dirname $0) +#protoc -I ${SRC_DIR}/proto/ ${SRC_DIR}/proto/downloader.proto --go_out=${SRC_DIR}/proto --go-grpc_out=${SRC_DIR}/proto + +docker run --platform linux/amd64 -v ${PWD}:/tmp ${PROTOC_IMAGE} /tmp/downloader.proto diff --git a/p2p/stream/protocols/sync/message/gen.sh b/p2p/stream/protocols/sync/message/gen.sh new file mode 100755 index 0000000000..1e61be7d4f --- /dev/null +++ b/p2p/stream/protocols/sync/message/gen.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker run --platform linux/amd64 -v ${PWD}:/tmp ${PROTOC_IMAGE} /tmp/msg.proto \ No newline at end of file diff --git a/p2p/stream/protocols/sync/message/generate.go b/p2p/stream/protocols/sync/message/generate.go index 881c9fc6f5..22f91a10e7 100644 --- a/p2p/stream/protocols/sync/message/generate.go +++ b/p2p/stream/protocols/sync/message/generate.go @@ -1,3 +1,3 @@ package message -//go:generate protoc msg.proto --go_out=. +//go:generate ./gen.sh diff --git a/p2p/stream/protocols/sync/message/msg.pb.go b/p2p/stream/protocols/sync/message/msg.pb.go index edc87badce..5928a7dea4 100644 --- a/p2p/stream/protocols/sync/message/msg.pb.go +++ b/p2p/stream/protocols/sync/message/msg.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.26.0 -// protoc v3.15.8 +// protoc-gen-go v1.30.0 +// protoc v3.12.4 // source: msg.proto package message diff --git a/scripts/gogenerate.sh b/scripts/gogenerate.sh index 9adb847d00..91c67661ab 100755 --- a/scripts/gogenerate.sh +++ b/scripts/gogenerate.sh @@ -6,4 +6,4 @@ case "${0}" in *) progdir=.;; esac git grep -l '^//go:generate ' -- '*.go' | \ - "${progdir}/xargs_by_dir.sh" go generate -v -x + PROTOC_IMAGE="frozen621/harmony-proto:latest" "${progdir}/xargs_by_dir.sh" go generate -v -x diff --git a/scripts/install_build_tools.sh b/scripts/install_build_tools.sh deleted file mode 100755 index abebcce3c6..0000000000 --- a/scripts/install_build_tools.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -set -eu - -unset -v progdir -case "${0}" in -/*) progdir="/";; -*/*) progdir="${0%/*}";; -*) progdir="."; -esac - -sed -n 's/^ _ "\([^"]*\)"$/\1/p' "${progdir}/../tools/tools.go" | \ - xargs "${progdir}/goget.sh" - -"${progdir}/install_protoc.sh" -V 3.15.8 diff --git a/scripts/install_protoc.sh b/scripts/install_protoc.sh deleted file mode 100755 index afb4f4866e..0000000000 --- a/scripts/install_protoc.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash -set -eu -unset -v opt OPTIND OPTARG version platform destdir -version= -platform= -destdir=/usr/local -usage() { - case $# in - [1-9]*) echo "$@" >&2;; - esac - cat <<- ENDEND - usage: install_protoc.sh [-P platform] [-d destdir] -V version - options: - -V version protobuf version - -P platform fetch and use given platform (default: autodetect) - -d destdir install into the given dir (default: /usr/local) - -h print this help - ENDEND - exit 64 -} - -while getopts :V:P:d:h opt -do - case "${opt}" in - V) version="${OPTARG}";; - P) platform="${OPTARG}";; - d) destdir="${OPTARG}";; - h) usage;; - "?") usage "unrecognized option -${OPTARG}";; - ":") usage "missing argument for -${OPTARG}";; - *) echo "unhandled option -${OPTARG}" >&2; exit 70;; - esac -done -shift $((${OPTIND} - 1)) -case "${version}" in -"") usage "protobuf version (-V) not specified";; -esac -case "${platform}" in -"") - platform=$(uname -s) - case "${platform}" in - Darwin) platform=osx;; - Linux) platform=linux;; - *) echo "unsupported OS name (${platform})" >&2; exit 69;; - esac - arch=$(uname -m) - case "${arch}" in - aarch64) arch=aarch_64;; - esac - platform="${platform}-${arch}" - ;; -esac -unset -v tmpdir -trap 'case "${tmpdir-}" in ?*) rm -rf "${tmpdir}";; esac' EXIT -tmpdir=$(mktemp -d) -cd "${tmpdir}" -unset -v filename url -filename="protoc-${version}-${platform}.zip" -url="https://github.com/protocolbuffers/protobuf/releases/download/v${version}/${filename}" -echo "Downloading protoc v${version} for ${platform}..." -curl -s -S -L -o "${filename}" "${url}" -echo "Downloaded as ${filename}; unzipping into ${destdir}..." -sudo unzip -o -d "${destdir}" "${filename}" -echo "protoc v${version} has been installed in ${destdir}." -sudo chmod +x "${destdir}/bin/protoc" -exit 0 From 81c2fb06c4b313e305518f40a12cb7f509381739 Mon Sep 17 00:00:00 2001 From: Gheis Mohammadi Date: Sat, 20 May 2023 10:02:25 +0800 Subject: [PATCH 407/420] fix stream log issue, replace print log error with warning (#4433) --- api/service/stagedstreamsync/staged_stream_sync.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/service/stagedstreamsync/staged_stream_sync.go b/api/service/stagedstreamsync/staged_stream_sync.go index e73edd622f..a03dd03b1a 100644 --- a/api/service/stagedstreamsync/staged_stream_sync.go +++ b/api/service/stagedstreamsync/staged_stream_sync.go @@ -403,7 +403,7 @@ func (s *StagedStreamSync) Run(db kv.RwDB, tx kv.RwTx, firstCycle bool) error { return err } if err := printLogs(tx, s.timings); err != nil { - return err + utils.Logger().Warn().Err(err).Msg("print timing logs failed") } s.currentStage = 0 return nil @@ -440,8 +440,8 @@ func printLogs(tx kv.RwTx, timings []Timing) error { } } if len(logCtx) > 0 { - utils.Logger().Info(). - Msgf(WrapStagedSyncMsg(fmt.Sprintf("Timings (slower than 50ms) %v", logCtx...))) + timingLog := fmt.Sprintf("Timings (slower than 50ms) %v", logCtx) + utils.Logger().Info().Msgf(WrapStagedSyncMsg(timingLog)) } if tx == nil { From c4427231a61ff1ec324a3ca21ead9947701d450c Mon Sep 17 00:00:00 2001 From: Max <82761650+MaxMustermann2@users.noreply.github.com> Date: Thu, 25 May 2023 15:33:54 +0000 Subject: [PATCH 408/420] hmy: Gas Price Oracle improvements (#4436) This change makes the gas price oracle options available as options in the configuration file / command line. In addition, the gas price oracle's suggestion mechanism has been modified to return the default gas price when block utilization is low. In other words, the oracle can be configured to return the 60th percentile gas price from the last 5 blocks with 3 transactions each, or return the default gas price if those 5 blocks were utilized less than 50% of their capacity Fixes #4357 and supersedes #4362 --- cmd/harmony/config_migrations.go | 27 ++++++ cmd/harmony/default.go | 12 ++- cmd/harmony/flags.go | 74 ++++++++++++++++ cmd/harmony/flags_test.go | 60 +++++++++++++ cmd/harmony/main.go | 3 +- hmy/gasprice.go | 125 +++++++++++++++++++--------- hmy/hmy.go | 8 +- internal/cli/flag.go | 23 ++++- internal/cli/parse.go | 15 ++++ internal/configs/harmony/harmony.go | 20 +++++ 10 files changed, 319 insertions(+), 48 deletions(-) diff --git a/cmd/harmony/config_migrations.go b/cmd/harmony/config_migrations.go index 84607c2d0d..8f222b3d62 100644 --- a/cmd/harmony/config_migrations.go +++ b/cmd/harmony/config_migrations.go @@ -378,6 +378,33 @@ func init() { return confTree } + migrations["2.5.14"] = func(confTree *toml.Tree) *toml.Tree { + if confTree.Get("GPO.Blocks") == nil { + confTree.Set("GPO.Blocks", defaultConfig.GPO.Blocks) + } + if confTree.Get("GPO.Transactions") == nil { + confTree.Set("GPO.Transactions", defaultConfig.GPO.Transactions) + } + if confTree.Get("GPO.Percentile") == nil { + confTree.Set("GPO.Percentile", defaultConfig.GPO.Percentile) + } + if confTree.Get("GPO.DefaultPrice") == nil { + confTree.Set("GPO.DefaultPrice", defaultConfig.GPO.DefaultPrice) + } + if confTree.Get("GPO.MaxPrice") == nil { + confTree.Set("GPO.MaxPrice", defaultConfig.GPO.MaxPrice) + } + if confTree.Get("GPO.LowUsageThreshold") == nil { + confTree.Set("GPO.LowUsageThreshold", defaultConfig.GPO.LowUsageThreshold) + } + if confTree.Get("GPO.BlockGasLimit") == nil { + confTree.Set("GPO.BlockGasLimit", defaultConfig.GPO.BlockGasLimit) + } + // upgrade minor version because of `GPO` section introduction + confTree.Set("Version", "2.6.0") + return confTree + } + // check that the latest version here is the same as in default.go largestKey := getNextVersion(migrations) if largestKey != tomlConfigVersion { diff --git a/cmd/harmony/default.go b/cmd/harmony/default.go index 2c15e123f7..38ae3dacbb 100644 --- a/cmd/harmony/default.go +++ b/cmd/harmony/default.go @@ -2,11 +2,12 @@ package main import ( "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/hmy" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" ) -const tomlConfigVersion = "2.5.14" +const tomlConfigVersion = "2.6.0" const ( defNetworkType = nodeconfig.Mainnet @@ -120,6 +121,15 @@ var defaultConfig = harmonyconfig.HarmonyConfig{ CacheTime: 10, CacheSize: 512, }, + GPO: harmonyconfig.GasPriceOracleConfig{ + Blocks: hmy.DefaultGPOConfig.Blocks, + Transactions: hmy.DefaultGPOConfig.Transactions, + Percentile: hmy.DefaultGPOConfig.Percentile, + DefaultPrice: hmy.DefaultGPOConfig.DefaultPrice, + MaxPrice: hmy.DefaultGPOConfig.MaxPrice, + LowUsageThreshold: hmy.DefaultGPOConfig.LowUsageThreshold, + BlockGasLimit: hmy.DefaultGPOConfig.BlockGasLimit, + }, } var defaultSysConfig = harmonyconfig.SysConfig{ diff --git a/cmd/harmony/flags.go b/cmd/harmony/flags.go index 9760652e03..acf53d8b3a 100644 --- a/cmd/harmony/flags.go +++ b/cmd/harmony/flags.go @@ -249,6 +249,16 @@ var ( cacheSizeFlag, } + gpoFlags = []cli.Flag{ + gpoBlocksFlag, + gpoTransactionsFlag, + gpoPercentileFlag, + gpoDefaultPriceFlag, + gpoMaxPriceFlag, + gpoLowUsageThresholdFlag, + gpoBlockGasLimitFlag, + } + metricsFlags = []cli.Flag{ metricsETHFlag, metricsExpensiveETHFlag, @@ -364,6 +374,7 @@ func getRootFlags() []cli.Flag { flags = append(flags, prometheusFlags...) flags = append(flags, syncFlags...) flags = append(flags, shardDataFlags...) + flags = append(flags, gpoFlags...) flags = append(flags, metricsFlags...) return flags @@ -1932,6 +1943,45 @@ var ( } ) +// gas price oracle flags +var ( + gpoBlocksFlag = cli.IntFlag{ + Name: "gpo.blocks", + Usage: "Number of recent blocks to check for gas prices", + DefValue: defaultConfig.GPO.Blocks, + } + gpoTransactionsFlag = cli.IntFlag{ + Name: "gpo.transactions", + Usage: "Number of transactions to sample in a block", + DefValue: defaultConfig.GPO.Transactions, + } + gpoPercentileFlag = cli.IntFlag{ + Name: "gpo.percentile", + Usage: "Suggested gas price is the given percentile of a set of recent transaction gas prices", + DefValue: defaultConfig.GPO.Percentile, + } + gpoDefaultPriceFlag = cli.Int64Flag{ + Name: "gpo.defaultprice", + Usage: "The gas price to suggest before data is available, and the price to suggest when block utilization is low", + DefValue: defaultConfig.GPO.DefaultPrice, + } + gpoMaxPriceFlag = cli.Int64Flag{ + Name: "gpo.maxprice", + Usage: "Maximum gasprice to be recommended by gpo", + DefValue: defaultConfig.GPO.MaxPrice, + } + gpoLowUsageThresholdFlag = cli.IntFlag{ + Name: "gpo.low-usage-threshold", + Usage: "The block usage threshold below which the default gas price is suggested (0 to disable)", + DefValue: defaultConfig.GPO.LowUsageThreshold, + } + gpoBlockGasLimitFlag = cli.IntFlag{ + Name: "gpo.block-gas-limit", + Usage: "The gas limit, per block. If set to 0, it is pulled from the block header", + DefValue: defaultConfig.GPO.BlockGasLimit, + } +) + // metrics flags required for the go-eth library // https://github.com/ethereum/go-ethereum/blob/master/metrics/metrics.go#L35-L55 var ( @@ -1965,3 +2015,27 @@ func applyShardDataFlags(cmd *cobra.Command, cfg *harmonyconfig.HarmonyConfig) { cfg.ShardData.CacheSize = cli.GetIntFlagValue(cmd, cacheSizeFlag) } } + +func applyGPOFlags(cmd *cobra.Command, cfg *harmonyconfig.HarmonyConfig) { + if cli.IsFlagChanged(cmd, gpoBlocksFlag) { + cfg.GPO.Blocks = cli.GetIntFlagValue(cmd, gpoBlocksFlag) + } + if cli.IsFlagChanged(cmd, gpoTransactionsFlag) { + cfg.GPO.Transactions = cli.GetIntFlagValue(cmd, gpoTransactionsFlag) + } + if cli.IsFlagChanged(cmd, gpoPercentileFlag) { + cfg.GPO.Percentile = cli.GetIntFlagValue(cmd, gpoPercentileFlag) + } + if cli.IsFlagChanged(cmd, gpoDefaultPriceFlag) { + cfg.GPO.DefaultPrice = cli.GetInt64FlagValue(cmd, gpoDefaultPriceFlag) + } + if cli.IsFlagChanged(cmd, gpoMaxPriceFlag) { + cfg.GPO.MaxPrice = cli.GetInt64FlagValue(cmd, gpoMaxPriceFlag) + } + if cli.IsFlagChanged(cmd, gpoLowUsageThresholdFlag) { + cfg.GPO.LowUsageThreshold = cli.GetIntFlagValue(cmd, gpoLowUsageThresholdFlag) + } + if cli.IsFlagChanged(cmd, gpoBlockGasLimitFlag) { + cfg.GPO.BlockGasLimit = cli.GetIntFlagValue(cmd, gpoBlockGasLimitFlag) + } +} diff --git a/cmd/harmony/flags_test.go b/cmd/harmony/flags_test.go index 5338b9f71e..054c804215 100644 --- a/cmd/harmony/flags_test.go +++ b/cmd/harmony/flags_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/harmony-one/harmony/common/denominations" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" "github.com/spf13/cobra" @@ -172,6 +173,15 @@ func TestHarmonyFlags(t *testing.T) { CacheTime: 10, CacheSize: 512, }, + GPO: harmonyconfig.GasPriceOracleConfig{ + Blocks: defaultConfig.GPO.Blocks, + Transactions: defaultConfig.GPO.Transactions, + Percentile: defaultConfig.GPO.Percentile, + DefaultPrice: defaultConfig.GPO.DefaultPrice, + MaxPrice: defaultConfig.GPO.MaxPrice, + LowUsageThreshold: defaultConfig.GPO.LowUsageThreshold, + BlockGasLimit: defaultConfig.GPO.BlockGasLimit, + }, }, }, } @@ -1350,6 +1360,56 @@ func TestSysFlags(t *testing.T) { } } +func TestGPOFlags(t *testing.T) { + tests := []struct { + args []string + expConfig harmonyconfig.GasPriceOracleConfig + expErr error + }{ + { + args: []string{}, + expConfig: harmonyconfig.GasPriceOracleConfig{ + Blocks: defaultConfig.GPO.Blocks, + Transactions: defaultConfig.GPO.Transactions, + Percentile: defaultConfig.GPO.Percentile, + DefaultPrice: defaultConfig.GPO.DefaultPrice, + MaxPrice: defaultConfig.GPO.MaxPrice, + LowUsageThreshold: defaultConfig.GPO.LowUsageThreshold, + BlockGasLimit: defaultConfig.GPO.BlockGasLimit, + }, + }, + { + args: []string{"--gpo.blocks", "5", "--gpo.transactions", "1", "--gpo.percentile", "2", "--gpo.defaultprice", "101000000000", "--gpo.maxprice", "400000000000", "--gpo.low-usage-threshold", "60", "--gpo.block-gas-limit", "10000000"}, + expConfig: harmonyconfig.GasPriceOracleConfig{ + Blocks: 5, + Transactions: 1, + Percentile: 2, + DefaultPrice: 101 * denominations.Nano, + MaxPrice: 400 * denominations.Nano, + LowUsageThreshold: 60, + BlockGasLimit: 10_000_000, + }, + }, + } + + for i, test := range tests { + ts := newFlagTestSuite(t, gpoFlags, applyGPOFlags) + hc, err := ts.run(test.args) + + if assErr := assertError(err, test.expErr); assErr != nil { + t.Fatalf("Test %v: %v", i, assErr) + } + if err != nil || test.expErr != nil { + continue + } + + if !reflect.DeepEqual(hc.GPO, test.expConfig) { + t.Errorf("Test %v:\n\t%+v\n\t%+v", i, hc.GPO, test.expConfig) + } + ts.tearDown() + } +} + func TestDevnetFlags(t *testing.T) { tests := []struct { args []string diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 94038749d4..e22eeb7a02 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -249,6 +249,7 @@ func applyRootFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { applyPrometheusFlags(cmd, config) applySyncFlags(cmd, config) applyShardDataFlags(cmd, config) + applyGPOFlags(cmd, config) } func setupNodeLog(config harmonyconfig.HarmonyConfig) { @@ -262,7 +263,7 @@ func setupNodeLog(config harmonyconfig.HarmonyConfig) { utils.SetLogContext(ip, strconv.Itoa(port)) } - if config.Log.Console != true { + if !config.Log.Console { utils.AddLogFile(logPath, config.Log.RotateSize, config.Log.RotateCount, config.Log.RotateMaxAge) } } diff --git a/hmy/gasprice.go b/hmy/gasprice.go index c4c14a87d4..9dcaca4685 100644 --- a/hmy/gasprice.go +++ b/hmy/gasprice.go @@ -18,13 +18,14 @@ package hmy import ( "context" - "fmt" "math/big" "sort" "sync" "github.com/harmony-one/harmony/block" + "github.com/harmony-one/harmony/common/denominations" "github.com/harmony-one/harmony/eth/rpc" + "github.com/harmony-one/harmony/internal/configs/harmony" "github.com/harmony-one/harmony/internal/utils" "github.com/ethereum/go-ethereum/common" @@ -32,17 +33,6 @@ import ( "github.com/harmony-one/harmony/core/types" ) -const sampleNumber = 3 // Number of transactions sampled in a block - -var DefaultMaxPrice = big.NewInt(1e12) // 1000 gwei is the max suggested limit - -type GasPriceConfig struct { - Blocks int - Percentile int - Default *big.Int `toml:",omitempty"` - MaxPrice *big.Int `toml:",omitempty"` -} - // OracleBackend includes all necessary background APIs for oracle. type OracleBackend interface { HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*block.Header, error) @@ -50,8 +40,7 @@ type OracleBackend interface { ChainConfig() *params.ChainConfig } -// Oracle recommends gas prices based on the content of recent -// blocks. Suitable for both light and full clients. +// Oracle recommends gas prices based on the content of recent blocks. type Oracle struct { backend *Harmony lastHead common.Hash @@ -60,38 +49,74 @@ type Oracle struct { cacheLock sync.RWMutex fetchLock sync.Mutex - checkBlocks int - percentile int + checkBlocks int + percentile int + checkTxs int + lowUsageThreshold float64 + blockGasLimit int + defaultPrice *big.Int +} + +var DefaultGPOConfig = harmony.GasPriceOracleConfig{ + Blocks: 20, + Transactions: 3, + Percentile: 60, + DefaultPrice: 100 * denominations.Nano, // 100 gwei + MaxPrice: 1000 * denominations.Nano, // 1000 gwei + LowUsageThreshold: 50, + BlockGasLimit: 0, // TODO should we set default to 30M? } // NewOracle returns a new gasprice oracle which can recommend suitable // gasprice for newly created transaction. -func NewOracle(backend *Harmony, params GasPriceConfig) *Oracle { +func NewOracle(backend *Harmony, params *harmony.GasPriceOracleConfig) *Oracle { blocks := params.Blocks if blocks < 1 { - blocks = 1 - utils.Logger().Warn().Msg(fmt.Sprint("Sanitizing invalid gasprice oracle sample blocks", "provided", params.Blocks, "updated", blocks)) + blocks = DefaultGPOConfig.Blocks + utils.Logger().Warn(). + Int("provided", params.Blocks). + Int("updated", blocks). + Msg("Sanitizing invalid gasprice oracle sample blocks") } - percent := params.Percentile - if percent < 0 { - percent = 0 - utils.Logger().Warn().Msg(fmt.Sprint("Sanitizing invalid gasprice oracle sample percentile", "provided", params.Percentile, "updated", percent)) + txs := params.Transactions + if txs < 1 { + txs = DefaultGPOConfig.Transactions + utils.Logger().Warn(). + Int("provided", params.Transactions). + Int("updated", txs). + Msg("Sanitizing invalid gasprice oracle sample transactions") } - if percent > 100 { - percent = 100 - utils.Logger().Warn().Msg(fmt.Sprint("Sanitizing invalid gasprice oracle sample percentile", "provided", params.Percentile, "updated", percent)) + percentile := params.Percentile + if percentile < 0 || percentile > 100 { + percentile = DefaultGPOConfig.Percentile + utils.Logger().Warn(). + Int("provided", params.Percentile). + Int("updated", percentile). + Msg("Sanitizing invalid gasprice oracle percentile") } - maxPrice := params.MaxPrice - if maxPrice == nil || maxPrice.Int64() <= 0 { - maxPrice = DefaultMaxPrice - utils.Logger().Warn().Msg(fmt.Sprint("Sanitizing invalid gasprice oracle price cap", "provided", params.MaxPrice, "updated", maxPrice)) + // no sanity check done, simply convert it + defaultPrice := big.NewInt(params.DefaultPrice) + maxPrice := big.NewInt(params.MaxPrice) + lowUsageThreshold := float64(params.LowUsageThreshold) / 100.0 + if lowUsageThreshold < 0 || lowUsageThreshold > 1 { + lowUsageThreshold = float64(DefaultGPOConfig.LowUsageThreshold) / 100.0 + utils.Logger().Warn(). + Float64("provided", float64(params.LowUsageThreshold)/100.0). + Float64("updated", lowUsageThreshold). + Msg("Sanitizing invalid gasprice oracle lowUsageThreshold") } + blockGasLimit := params.BlockGasLimit return &Oracle{ - backend: backend, - lastPrice: params.Default, - maxPrice: maxPrice, - checkBlocks: blocks, - percentile: percent, + backend: backend, + lastPrice: defaultPrice, + maxPrice: maxPrice, + checkBlocks: blocks, + percentile: percentile, + checkTxs: txs, + lowUsageThreshold: lowUsageThreshold, + blockGasLimit: blockGasLimit, + // do not reference lastPrice + defaultPrice: new(big.Int).Set(defaultPrice), } } @@ -124,9 +149,10 @@ func (gpo *Oracle) SuggestPrice(ctx context.Context) (*big.Int, error) { result = make(chan getBlockPricesResult, gpo.checkBlocks) quit = make(chan struct{}) txPrices []*big.Int + usageSum float64 ) for sent < gpo.checkBlocks && number > 0 { - go gpo.getBlockPrices(ctx, types.MakeSigner(gpo.backend.ChainConfig(), big.NewInt(int64(number))), number, sampleNumber, result, quit) + go gpo.getBlockPrices(ctx, types.MakeSigner(gpo.backend.ChainConfig(), big.NewInt(int64(number))), number, gpo.checkTxs, result, quit) sent++ exp++ number-- @@ -149,18 +175,26 @@ func (gpo *Oracle) SuggestPrice(ctx context.Context) (*big.Int, error) { // meaningful returned, try to query more blocks. But the maximum // is 2*checkBlocks. if len(res.prices) == 1 && len(txPrices)+1+exp < gpo.checkBlocks*2 && number > 0 { - go gpo.getBlockPrices(ctx, types.MakeSigner(gpo.backend.ChainConfig(), big.NewInt(int64(number))), number, sampleNumber, result, quit) + go gpo.getBlockPrices(ctx, types.MakeSigner(gpo.backend.ChainConfig(), big.NewInt(int64(number))), number, gpo.checkTxs, result, quit) sent++ exp++ number-- } txPrices = append(txPrices, res.prices...) + usageSum += res.usage } price := lastPrice if len(txPrices) > 0 { sort.Sort(bigIntArray(txPrices)) price = txPrices[(len(txPrices)-1)*gpo.percentile/100] } + // `sent` is the number of queries that are sent, while `exp` and `number` count down at query resolved, and sent respectively + // each query is per block, therefore `sent` is the number of blocks for which the usage was (successfully) determined + // approximation that only holds when the gas limits, of all blocks that are sampled, are equal + usage := usageSum / float64(sent) + if usage < gpo.lowUsageThreshold { + price = new(big.Int).Set(gpo.defaultPrice) + } if price.Cmp(gpo.maxPrice) > 0 { price = new(big.Int).Set(gpo.maxPrice) } @@ -173,6 +207,7 @@ func (gpo *Oracle) SuggestPrice(ctx context.Context) (*big.Int, error) { type getBlockPricesResult struct { prices []*big.Int + usage float64 err error } @@ -192,7 +227,9 @@ func (gpo *Oracle) getBlockPrices(ctx context.Context, signer types.Signer, bloc block, err := gpo.backend.BlockByNumber(ctx, rpc.BlockNumber(blockNum)) if block == nil { select { - case result <- getBlockPricesResult{nil, err}: + // when no block is found, `err` is returned which short-circuits the oracle entirely + // therefore the `usage` becomes irrelevant + case result <- getBlockPricesResult{nil, 2.0, err}: case <-quit: } return @@ -212,8 +249,18 @@ func (gpo *Oracle) getBlockPrices(ctx context.Context, signer types.Signer, bloc } } } + // HACK + var gasLimit float64 + if gpo.blockGasLimit == 0 { + gasLimit = float64(block.GasLimit()) + } else { + gasLimit = float64(gpo.blockGasLimit) + } + // if `gasLimit` is 0, no crash. +Inf is returned and percentile is applied + // this usage includes any transactions from the miner, which are excluded by the `prices` slice + usage := float64(block.GasUsed()) / gasLimit select { - case result <- getBlockPricesResult{prices, nil}: + case result <- getBlockPricesResult{prices, usage, nil}: case <-quit: } } diff --git a/hmy/hmy.go b/hmy/hmy.go index fd01f123c6..24f0caa127 100644 --- a/hmy/hmy.go +++ b/hmy/hmy.go @@ -157,12 +157,8 @@ func New( } // Setup gas price oracle - gpoParams := GasPriceConfig{ - Blocks: 20, // take all eligible txs past 20 blocks and sort them - Percentile: 80, // get the 80th percentile when sorted in an ascending manner - Default: big.NewInt(100e9), // minimum of 100 gwei - } - gpo := NewOracle(backend, gpoParams) + config := nodeAPI.GetConfig().HarmonyConfig.GPO + gpo := NewOracle(backend, &config) backend.gpo = gpo return backend diff --git a/internal/cli/flag.go b/internal/cli/flag.go index f178789bd1..755a6a73b9 100644 --- a/internal/cli/flag.go +++ b/internal/cli/flag.go @@ -3,7 +3,9 @@ package cli -import "github.com/spf13/pflag" +import ( + "github.com/spf13/pflag" +) // Flag is the interface for cli flags. // To get the value after cli parsing, use fs.GetString(flag.Name) @@ -62,6 +64,23 @@ func (f IntFlag) RegisterTo(fs *pflag.FlagSet) error { return markHiddenOrDeprecated(fs, f.Name, f.Deprecated, f.Hidden) } +// Int64Flag is the flag with int64 value, used for gwei configurations +type Int64Flag struct { + Name string + Shorthand string + Usage string + Deprecated string + Hidden bool + + DefValue int64 +} + +// RegisterTo register the int flag to FlagSet +func (f Int64Flag) RegisterTo(fs *pflag.FlagSet) error { + fs.Int64P(f.Name, f.Shorthand, f.DefValue, f.Usage) + return markHiddenOrDeprecated(fs, f.Name, f.Deprecated, f.Hidden) +} + // StringSliceFlag is the flag with string slice value type StringSliceFlag struct { Name string @@ -122,6 +141,8 @@ func getFlagName(flag Flag) string { return f.Name case IntSliceFlag: return f.Name + case Int64Flag: + return f.Name } return "" } diff --git a/internal/cli/parse.go b/internal/cli/parse.go index bc9565a3e3..326c923ad5 100644 --- a/internal/cli/parse.go +++ b/internal/cli/parse.go @@ -70,6 +70,12 @@ func GetIntFlagValue(cmd *cobra.Command, flag IntFlag) int { return getIntFlagValue(cmd.Flags(), flag) } +// GetInt64FlagValue get the int value for the given Int64Flag from the local flags of the +// cobra command. +func GetInt64FlagValue(cmd *cobra.Command, flag Int64Flag) int64 { + return getInt64FlagValue(cmd.Flags(), flag) +} + // GetIntPersistentFlagValue get the int value for the given IntFlag from the persistent // flags of the cobra command. func GetIntPersistentFlagValue(cmd *cobra.Command, flag IntFlag) int { @@ -85,6 +91,15 @@ func getIntFlagValue(fs *pflag.FlagSet, flag IntFlag) int { return val } +func getInt64FlagValue(fs *pflag.FlagSet, flag Int64Flag) int64 { + val, err := fs.GetInt64(flag.Name) + if err != nil { + handleParseError(err) + return 0 + } + return val +} + // GetStringSliceFlagValue get the string slice value for the given StringSliceFlag from // the local flags of the cobra command. func GetStringSliceFlagValue(cmd *cobra.Command, flag StringSliceFlag) []string { diff --git a/internal/configs/harmony/harmony.go b/internal/configs/harmony/harmony.go index 32161d1deb..f4f2ddb96f 100644 --- a/internal/configs/harmony/harmony.go +++ b/internal/configs/harmony/harmony.go @@ -35,6 +35,7 @@ type HarmonyConfig struct { TiKV *TiKVConfig `toml:",omitempty"` DNSSync DnsSync ShardData ShardDataConfig + GPO GasPriceOracleConfig } func (hc HarmonyConfig) ToRPCServerConfig() nodeconfig.RPCServerConfig { @@ -157,6 +158,25 @@ type ShardDataConfig struct { CacheSize int } +type GasPriceOracleConfig struct { + // the number of blocks to sample + Blocks int + // the number of transactions to sample, per block + Transactions int + // the percentile to pick from there + Percentile int + // the default gas price, if the above data is not available + DefaultPrice int64 + // the maximum suggested gas price + MaxPrice int64 + // when block usage (gas) for last `Blocks` blocks is below `LowUsageThreshold`, + // we return the Default price + LowUsageThreshold int + // hack: our block header reports an 80m gas limit, but it is actually 30M. + // if set to non-zero, this is applied UNCHECKED + BlockGasLimit int +} + type ConsensusConfig struct { MinPeers int AggregateSig bool From 3d4bf3f49c7afdc4ead7996ed36aca38c3445d86 Mon Sep 17 00:00:00 2001 From: Gheis Mohammadi Date: Tue, 30 May 2023 13:33:11 +0800 Subject: [PATCH 409/420] change cache folder in sync and move it under data folder (#4438) --- api/service/stagedstreamsync/downloader.go | 4 ++-- api/service/stagedstreamsync/downloaders.go | 4 ++-- api/service/stagedstreamsync/service.go | 4 ++-- api/service/stagedstreamsync/syncing.go | 16 +++++++++------- api/service/stagedsync/stage_blockhashes.go | 10 ++++++---- api/service/stagedsync/stage_bodies.go | 10 ++++++---- api/service/stagedsync/syncing.go | 12 ++++++++---- cmd/harmony/main.go | 3 +-- node/node_syncing.go | 3 ++- 9 files changed, 38 insertions(+), 28 deletions(-) diff --git a/api/service/stagedstreamsync/downloader.go b/api/service/stagedstreamsync/downloader.go index a20b4ac79a..668698f107 100644 --- a/api/service/stagedstreamsync/downloader.go +++ b/api/service/stagedstreamsync/downloader.go @@ -37,7 +37,7 @@ type ( ) // NewDownloader creates a new downloader -func NewDownloader(host p2p.Host, bc core.BlockChain, isBeaconNode bool, config Config) *Downloader { +func NewDownloader(host p2p.Host, bc core.BlockChain, dbDir string, isBeaconNode bool, config Config) *Downloader { config.fixValues() sp := sync.NewProtocol(sync.Config{ @@ -68,7 +68,7 @@ func NewDownloader(host p2p.Host, bc core.BlockChain, isBeaconNode bool, config ctx, cancel := context.WithCancel(context.Background()) //TODO: use mem db should be in config file - stagedSyncInstance, err := CreateStagedSync(ctx, bc, false, isBeaconNode, sp, config, logger, config.LogProgress) + stagedSyncInstance, err := CreateStagedSync(ctx, bc, dbDir, false, isBeaconNode, sp, config, logger, config.LogProgress) if err != nil { cancel() return nil diff --git a/api/service/stagedstreamsync/downloaders.go b/api/service/stagedstreamsync/downloaders.go index 0e79c79631..2df8b74aab 100644 --- a/api/service/stagedstreamsync/downloaders.go +++ b/api/service/stagedstreamsync/downloaders.go @@ -15,7 +15,7 @@ type Downloaders struct { } // NewDownloaders creates Downloaders for sync of multiple blockchains -func NewDownloaders(host p2p.Host, bcs []core.BlockChain, config Config) *Downloaders { +func NewDownloaders(host p2p.Host, bcs []core.BlockChain, dbDir string, config Config) *Downloaders { ds := make(map[uint32]*Downloader) isBeaconNode := len(bcs) == 1 for _, bc := range bcs { @@ -25,7 +25,7 @@ func NewDownloaders(host p2p.Host, bcs []core.BlockChain, config Config) *Downlo if _, ok := ds[bc.ShardID()]; ok { continue } - ds[bc.ShardID()] = NewDownloader(host, bc, isBeaconNode, config) + ds[bc.ShardID()] = NewDownloader(host, bc, dbDir, isBeaconNode, config) } return &Downloaders{ ds: ds, diff --git a/api/service/stagedstreamsync/service.go b/api/service/stagedstreamsync/service.go index 46b182fb53..40fbf7097c 100644 --- a/api/service/stagedstreamsync/service.go +++ b/api/service/stagedstreamsync/service.go @@ -11,9 +11,9 @@ type StagedStreamSyncService struct { } // NewService creates a new downloader service -func NewService(host p2p.Host, bcs []core.BlockChain, config Config) *StagedStreamSyncService { +func NewService(host p2p.Host, bcs []core.BlockChain, config Config, dbDir string) *StagedStreamSyncService { return &StagedStreamSyncService{ - Downloaders: NewDownloaders(host, bcs, config), + Downloaders: NewDownloaders(host, bcs, dbDir, config), } } diff --git a/api/service/stagedstreamsync/syncing.go b/api/service/stagedstreamsync/syncing.go index de9b884816..98b74cf0d0 100644 --- a/api/service/stagedstreamsync/syncing.go +++ b/api/service/stagedstreamsync/syncing.go @@ -3,6 +3,7 @@ package stagedstreamsync import ( "context" "fmt" + "path/filepath" "sync" "time" @@ -37,6 +38,7 @@ var Buckets = []string{ // CreateStagedSync creates an instance of staged sync func CreateStagedSync(ctx context.Context, bc core.BlockChain, + dbDir string, UseMemDB bool, isBeaconNode bool, protocol syncProtocol, @@ -55,9 +57,9 @@ func CreateStagedSync(ctx context.Context, dbs[i] = memdb.New() } } else { - mainDB = mdbx.NewMDBX(log.New()).Path(GetBlockDbPath(isBeacon, -1)).MustOpen() + mainDB = mdbx.NewMDBX(log.New()).Path(GetBlockDbPath(isBeacon, -1, dbDir)).MustOpen() for i := 0; i < config.Concurrency; i++ { - dbPath := GetBlockDbPath(isBeacon, i) + dbPath := GetBlockDbPath(isBeacon, i, dbDir) dbs[i] = mdbx.NewMDBX(log.New()).Path(dbPath).MustOpen() } } @@ -137,18 +139,18 @@ func initDB(ctx context.Context, mainDB kv.RwDB, dbs []kv.RwDB, concurrency int) } // GetBlockDbPath returns the path of the cache database which stores blocks -func GetBlockDbPath(beacon bool, loopID int) string { +func GetBlockDbPath(beacon bool, loopID int, dbDir string) string { if beacon { if loopID >= 0 { - return fmt.Sprintf("%s_%d", "cache/beacon_blocks_db", loopID) + return fmt.Sprintf("%s_%d", filepath.Join(dbDir, "cache/beacon_blocks_db"), loopID) } else { - return "cache/beacon_blocks_db_main" + return filepath.Join(dbDir, "cache/beacon_blocks_db_main") } } else { if loopID >= 0 { - return fmt.Sprintf("%s_%d", "cache/blocks_db", loopID) + return fmt.Sprintf("%s_%d", filepath.Join(dbDir, "cache/blocks_db"), loopID) } else { - return "cache/blocks_db_main" + return filepath.Join(dbDir, "cache/blocks_db_main") } } } diff --git a/api/service/stagedsync/stage_blockhashes.go b/api/service/stagedsync/stage_blockhashes.go index 1f049d52a9..61e0718dbd 100644 --- a/api/service/stagedsync/stage_blockhashes.go +++ b/api/service/stagedsync/stage_blockhashes.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "fmt" + "path/filepath" "strconv" "time" @@ -37,8 +38,8 @@ func NewStageBlockHashes(cfg StageBlockHashesCfg) *StageBlockHashes { } } -func NewStageBlockHashesCfg(ctx context.Context, bc core.BlockChain, db kv.RwDB, isBeacon bool, turbo bool, logProgress bool) StageBlockHashesCfg { - cachedb, err := initHashesCacheDB(ctx, isBeacon) +func NewStageBlockHashesCfg(ctx context.Context, bc core.BlockChain, dbDir string, db kv.RwDB, isBeacon bool, turbo bool, logProgress bool) StageBlockHashesCfg { + cachedb, err := initHashesCacheDB(ctx, dbDir, isBeacon) if err != nil { panic("can't initialize sync caches") } @@ -53,13 +54,14 @@ func NewStageBlockHashesCfg(ctx context.Context, bc core.BlockChain, db kv.RwDB, } } -func initHashesCacheDB(ctx context.Context, isBeacon bool) (db kv.RwDB, err error) { +func initHashesCacheDB(ctx context.Context, dbDir string, isBeacon bool) (db kv.RwDB, err error) { // create caches db cachedbName := BlockHashesCacheDB if isBeacon { cachedbName = "beacon_" + cachedbName } - cachedb := mdbx.NewMDBX(log.New()).Path(cachedbName).MustOpen() + dbPath := filepath.Join(dbDir, cachedbName) + cachedb := mdbx.NewMDBX(log.New()).Path(dbPath).MustOpen() // create transaction on cachedb tx, errRW := cachedb.BeginRw(ctx) if errRW != nil { diff --git a/api/service/stagedsync/stage_bodies.go b/api/service/stagedsync/stage_bodies.go index 5f0d4a5827..c3f91f0154 100644 --- a/api/service/stagedsync/stage_bodies.go +++ b/api/service/stagedsync/stage_bodies.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "fmt" + "path/filepath" "strconv" "sync" "time" @@ -37,8 +38,8 @@ func NewStageBodies(cfg StageBodiesCfg) *StageBodies { } } -func NewStageBodiesCfg(ctx context.Context, bc core.BlockChain, db kv.RwDB, isBeacon bool, turbo bool, logProgress bool) StageBodiesCfg { - cachedb, err := initBlocksCacheDB(ctx, isBeacon) +func NewStageBodiesCfg(ctx context.Context, bc core.BlockChain, dbDir string, db kv.RwDB, isBeacon bool, turbo bool, logProgress bool) StageBodiesCfg { + cachedb, err := initBlocksCacheDB(ctx, dbDir, isBeacon) if err != nil { panic("can't initialize sync caches") } @@ -53,13 +54,14 @@ func NewStageBodiesCfg(ctx context.Context, bc core.BlockChain, db kv.RwDB, isBe } } -func initBlocksCacheDB(ctx context.Context, isBeacon bool) (db kv.RwDB, err error) { +func initBlocksCacheDB(ctx context.Context, dbDir string, isBeacon bool) (db kv.RwDB, err error) { // create caches db cachedbName := BlockCacheDB if isBeacon { cachedbName = "beacon_" + cachedbName } - cachedb := mdbx.NewMDBX(log.New()).Path(cachedbName).MustOpen() + dbPath := filepath.Join(dbDir, cachedbName) + cachedb := mdbx.NewMDBX(log.New()).Path(dbPath).MustOpen() tx, errRW := cachedb.BeginRw(ctx) if errRW != nil { utils.Logger().Error(). diff --git a/api/service/stagedsync/syncing.go b/api/service/stagedsync/syncing.go index d204971573..11147f6a62 100644 --- a/api/service/stagedsync/syncing.go +++ b/api/service/stagedsync/syncing.go @@ -3,6 +3,7 @@ package stagedsync import ( "context" "fmt" + "path/filepath" "time" "github.com/c2h5oh/datasize" @@ -50,6 +51,7 @@ func CreateStagedSync( port string, peerHash [20]byte, bc core.BlockChain, + dbDir string, role nodeconfig.Role, isExplorer bool, TurboMode bool, @@ -82,9 +84,11 @@ func CreateStagedSync( db = mdbx.NewMDBX(log.New()).MapSize(dbMapSize).InMem("cache_db").MustOpen() } else { if isBeacon { - db = mdbx.NewMDBX(log.New()).Path("cache_beacon_db").MustOpen() + dbPath := filepath.Join(dbDir, "cache_beacon_db") + db = mdbx.NewMDBX(log.New()).Path(dbPath).MustOpen() } else { - db = mdbx.NewMDBX(log.New()).Path("cache_shard_db").MustOpen() + dbPath := filepath.Join(dbDir, "cache_shard_db") + db = mdbx.NewMDBX(log.New()).Path(dbPath).MustOpen() } } @@ -93,8 +97,8 @@ func CreateStagedSync( } headsCfg := NewStageHeadersCfg(ctx, bc, db) - blockHashesCfg := NewStageBlockHashesCfg(ctx, bc, db, isBeacon, TurboMode, logProgress) - bodiesCfg := NewStageBodiesCfg(ctx, bc, db, isBeacon, TurboMode, logProgress) + blockHashesCfg := NewStageBlockHashesCfg(ctx, bc, dbDir, db, isBeacon, TurboMode, logProgress) + bodiesCfg := NewStageBodiesCfg(ctx, bc, dbDir, db, isBeacon, TurboMode, logProgress) statesCfg := NewStageStatesCfg(ctx, bc, db, logProgress) lastMileCfg := NewStageLastMileCfg(ctx, bc, db) finishCfg := NewStageFinishCfg(ctx, db) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index e22eeb7a02..4e63ad4478 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -939,9 +939,8 @@ func setupStagedSyncService(node *node.Node, host p2p.Host, hc harmonyconfig.Har InsertHook: node.BeaconSyncHook, } } - //Setup stream sync service - s := stagedstreamsync.NewService(host, blockchains, sConfig) + s := stagedstreamsync.NewService(host, blockchains, sConfig, hc.General.DataDir) node.RegisterService(service.StagedStreamSync, s) diff --git a/node/node_syncing.go b/node/node_syncing.go index c33dd02c22..84eb1256ff 100644 --- a/node/node_syncing.go +++ b/node/node_syncing.go @@ -109,9 +109,10 @@ func (node *Node) createStagedSync(bc core.BlockChain) *stagedsync.StagedSync { mutatedPort := strconv.Itoa(mySyncPort + legacysync.SyncingPortDifference) role := node.NodeConfig.Role() isExplorer := node.NodeConfig.Role() == nodeconfig.ExplorerNode + dbDir := node.NodeConfig.DBDir if s, err := stagedsync.CreateStagedSync(node.SelfPeer.IP, mutatedPort, - node.GetSyncID(), bc, role, isExplorer, + node.GetSyncID(), bc, dbDir, role, isExplorer, node.NodeConfig.StagedSyncTurboMode, node.NodeConfig.UseMemDB, node.NodeConfig.DoubleCheckBlockHashes, From 78d802428c93d4bc94b8603cb899c159fe3add96 Mon Sep 17 00:00:00 2001 From: Konstantin <355847+Frozen@users.noreply.github.com> Date: Wed, 31 May 2023 17:34:52 -0400 Subject: [PATCH 410/420] Consensus: reduce consensus initialization steps. (#4387) * Removed hardcoded blockchain. * Fix for consensus. * Fix tests. * Fixed panic. * Clean up code. * Fix formatting. * Fixed comment. * Removed unused. * Fix merge issue. --- cmd/harmony/main.go | 13 ++++--- consensus/consensus.go | 23 ++++++++++--- consensus/consensus_service.go | 36 +++++++++++++++++++- consensus/consensus_service_test.go | 7 ++-- consensus/consensus_v2.go | 2 +- consensus/construct_test.go | 13 ++++--- consensus/downloader.go | 9 ++++- consensus/leader.go | 2 +- consensus/signature/signature.go | 8 ++--- consensus/threshold.go | 2 +- consensus/validator.go | 2 +- core/blockchain.go | 7 ---- core/blockchain_impl.go | 8 ++--- core/offchain.go | 4 +-- internal/chain/engine.go | 2 +- internal/chain/engine_test.go | 2 +- internal/registry/registry.go | 43 +++++++++++++++++++++-- node/node_explorer.go | 2 +- node/node_handler.go | 39 ++------------------- node/node_handler_test.go | 6 ++-- node/node_newblock_test.go | 9 ++--- staking/slash/double-sign.go | 2 +- staking/slash/double-sign_test.go | 3 +- staking/verify/verify.go | 53 ----------------------------- 24 files changed, 150 insertions(+), 147 deletions(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 4e63ad4478..2424791f0d 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -722,11 +722,12 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi // We are not beacon chain, make sure beacon already initialized. if nodeConfig.ShardID != shard.BeaconChainShardID { - _, err = collection.ShardChain(shard.BeaconChainShardID, core.Options{EpochChain: true}) + beacon, err := collection.ShardChain(shard.BeaconChainShardID, core.Options{EpochChain: true}) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "Error :%v \n", err) os.Exit(1) } + registry.SetBeaconchain(beacon) } blockchain, err = collection.ShardChain(nodeConfig.ShardID) @@ -734,11 +735,16 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi _, _ = fmt.Fprintf(os.Stderr, "Error :%v \n", err) os.Exit(1) } + registry.SetBlockchain(blockchain) + registry.SetWebHooks(nodeConfig.WebHooks.Hooks) + if registry.GetBeaconchain() == nil { + registry.SetBeaconchain(registry.GetBlockchain()) + } // Consensus object. decider := quorum.NewDecider(quorum.SuperMajorityVote, nodeConfig.ShardID) currentConsensus, err := consensus.New( - myHost, nodeConfig.ShardID, nodeConfig.ConsensusPriKey, registry.SetBlockchain(blockchain), decider, minPeers, aggregateSig) + myHost, nodeConfig.ShardID, nodeConfig.ConsensusPriKey, registry, decider, minPeers, aggregateSig) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "Error :%v \n", err) @@ -794,9 +800,6 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi Uint64("viewID", viewID). Msg("Init Blockchain") - // Assign closure functions to the consensus object - currentConsensus.SetBlockVerifier( - node.VerifyNewBlock(currentNode.NodeConfig, currentNode.Blockchain(), currentNode.Beaconchain())) currentConsensus.PostConsensusJob = currentNode.PostConsensusProcessing // update consensus information based on the blockchain currentConsensus.SetMode(currentConsensus.UpdateConsensusInformation()) diff --git a/consensus/consensus.go b/consensus/consensus.go index c5573c972d..ca2890fbbb 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -6,6 +6,7 @@ import ( "sync/atomic" "time" + "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/registry" @@ -139,14 +140,20 @@ func (consensus *Consensus) Blockchain() core.BlockChain { return consensus.registry.GetBlockchain() } -func (consensus *Consensus) ReadySignal(p ProposalType) { - consensus.readySignal <- p +// ChainReader returns the chain reader. +// This is mostly the same as Blockchain, but it returns only read methods, so we assume it's safe for concurrent use. +func (consensus *Consensus) ChainReader() engine.ChainReader { + return consensus.Blockchain() } func (consensus *Consensus) GetReadySignal() chan ProposalType { return consensus.readySignal } +func (consensus *Consensus) ReadySignal(p ProposalType) { + consensus.readySignal <- p +} + func (consensus *Consensus) CommitSigChannel() chan []byte { return consensus.commitSigChannel } @@ -155,6 +162,11 @@ func (consensus *Consensus) GetCommitSigChannel() chan []byte { return consensus.commitSigChannel } +// Beaconchain returns the beaconchain. +func (consensus *Consensus) Beaconchain() core.BlockChain { + return consensus.registry.GetBeaconchain() +} + // VerifyBlock is a function used to verify the block and keep trace of verified blocks. func (consensus *Consensus) verifyBlock(block *types.Block) error { if !consensus.FBFTLog.IsBlockVerified(block.Hash()) { @@ -232,8 +244,8 @@ func (consensus *Consensus) getConsensusLeaderPrivateKey() (*bls.PrivateKeyWrapp return consensus.getLeaderPrivateKey(consensus.LeaderPubKey.Object) } -// SetBlockVerifier sets the block verifier -func (consensus *Consensus) SetBlockVerifier(verifier VerifyBlockFunc) { +// setBlockVerifier sets the block verifier +func (consensus *Consensus) setBlockVerifier(verifier VerifyBlockFunc) { consensus.mutex.Lock() defer consensus.mutex.Unlock() consensus.BlockVerifier = verifier @@ -288,7 +300,6 @@ func New( // the blockchain during initialization as it was // displayed on explorer as Height right now consensus.SetCurBlockViewID(0) - consensus.ShardID = shard consensus.SlashChan = make(chan slash.Record) consensus.readySignal = make(chan ProposalType) consensus.commitSigChannel = make(chan []byte) @@ -297,6 +308,8 @@ func New( consensus.IgnoreViewIDCheck = abool.NewBool(false) // Make Sure Verifier is not null consensus.vc = newViewChange() + // TODO: reference to blockchain/beaconchain should be removed. + consensus.setBlockVerifier(VerifyNewBlock(registry.GetWebHooks(), consensus.Blockchain(), consensus.Beaconchain())) // init prometheus metrics initMetrics() diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index e8d7e1645c..2b80ecd8bc 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -5,9 +5,11 @@ import ( "sync/atomic" "time" + "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/multibls" + "github.com/harmony-one/harmony/webhooks" "github.com/ethereum/go-ethereum/common" protobuf "github.com/golang/protobuf/proto" @@ -590,7 +592,7 @@ func (consensus *Consensus) selfCommit(payload []byte) error { consensus.switchPhase("selfCommit", FBFTCommit) consensus.aggregatedPrepareSig = aggSig consensus.prepareBitmap = mask - commitPayload := signature.ConstructCommitPayload(consensus.Blockchain(), + commitPayload := signature.ConstructCommitPayload(consensus.ChainReader().Config(), block.Epoch(), block.Hash(), block.NumberU64(), block.Header().ViewID().Uint64()) for i, key := range consensus.priKey { if err := consensus.commitBitmap.SetKey(key.Pub.Bytes, true); err != nil { @@ -658,3 +660,35 @@ func (consensus *Consensus) getLogger() *zerolog.Logger { Logger() return &logger } + +// VerifyNewBlock is called by consensus participants to verify the block (account model) they are +// running consensus on. +func VerifyNewBlock(hooks *webhooks.Hooks, blockChain core.BlockChain, beaconChain core.BlockChain) func(*types.Block) error { + return func(newBlock *types.Block) error { + if err := blockChain.ValidateNewBlock(newBlock, beaconChain); err != nil { + if hooks := hooks; hooks != nil { + if p := hooks.ProtocolIssues; p != nil { + url := p.OnCannotCommit + go func() { + webhooks.DoPost(url, map[string]interface{}{ + "bad-header": newBlock.Header(), + "reason": err.Error(), + }) + }() + } + } + utils.Logger().Error(). + Str("blockHash", newBlock.Hash().Hex()). + Int("numTx", len(newBlock.Transactions())). + Int("numStakingTx", len(newBlock.StakingTransactions())). + Err(err). + Msg("[VerifyNewBlock] Cannot Verify New Block!!!") + return errors.Errorf( + "[VerifyNewBlock] Cannot Verify New Block!!! block-hash %s txn-count %d", + newBlock.Hash().Hex(), + len(newBlock.Transactions()), + ) + } + return nil + } +} diff --git a/consensus/consensus_service_test.go b/consensus/consensus_service_test.go index dd2fca7abc..00b8bd3462 100644 --- a/consensus/consensus_service_test.go +++ b/consensus/consensus_service_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/harmony-one/harmony/crypto/bls" + "github.com/harmony-one/harmony/internal/registry" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus/quorum" @@ -25,7 +26,8 @@ func TestSignAndMarshalConsensusMessage(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote, shard.BeaconChainShardID) blsPriKey := bls.RandPrivateKey() - consensus, err := New(host, shard.BeaconChainShardID, multibls.GetPrivateKeys(blsPriKey), nil, decider, 3, false) + reg := registry.New() + consensus, err := New(host, shard.BeaconChainShardID, multibls.GetPrivateKeys(blsPriKey), reg, decider, 3, false) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) } @@ -57,8 +59,9 @@ func TestSetViewID(t *testing.T) { quorum.SuperMajorityVote, shard.BeaconChainShardID, ) blsPriKey := bls.RandPrivateKey() + reg := registry.New() consensus, err := New( - host, shard.BeaconChainShardID, multibls.GetPrivateKeys(blsPriKey), nil, decider, 3, false, + host, shard.BeaconChainShardID, multibls.GetPrivateKeys(blsPriKey), reg, decider, 3, false, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 41d5b2d261..262cbe37d6 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -608,7 +608,7 @@ func (consensus *Consensus) verifyLastCommitSig(lastCommitSig []byte, blk *types } aggPubKey := consensus.commitBitmap.AggregatePublic - commitPayload := signature.ConstructCommitPayload(consensus.Blockchain(), + commitPayload := signature.ConstructCommitPayload(consensus.Blockchain().Config(), blk.Epoch(), blk.Hash(), blk.NumberU64(), blk.Header().ViewID().Uint64()) if !aggSig.VerifyHash(aggPubKey, commitPayload) { diff --git a/consensus/construct_test.go b/consensus/construct_test.go index 1add32219d..7188ebea68 100644 --- a/consensus/construct_test.go +++ b/consensus/construct_test.go @@ -10,6 +10,7 @@ import ( msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/crypto/bls" + "github.com/harmony-one/harmony/internal/registry" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" @@ -31,7 +32,8 @@ func TestConstructAnnounceMessage(test *testing.T) { quorum.SuperMajorityVote, shard.BeaconChainShardID, ) blsPriKey := bls.RandPrivateKey() - consensus, err := New(host, shard.BeaconChainShardID, multibls.GetPrivateKeys(blsPriKey), nil, decider, 3, false) + reg := registry.New() + consensus, err := New(host, shard.BeaconChainShardID, multibls.GetPrivateKeys(blsPriKey), reg, decider, 3, false) if err != nil { test.Fatalf("Cannot create consensus: %v", err) } @@ -63,7 +65,8 @@ func TestConstructPreparedMessage(test *testing.T) { quorum.SuperMajorityVote, shard.BeaconChainShardID, ) blsPriKey := bls.RandPrivateKey() - consensus, err := New(host, shard.BeaconChainShardID, multibls.GetPrivateKeys(blsPriKey), nil, decider, 3, false) + reg := registry.New() + consensus, err := New(host, shard.BeaconChainShardID, multibls.GetPrivateKeys(blsPriKey), reg, decider, 3, false) if err != nil { test.Fatalf("Cannot craeate consensus: %v", err) } @@ -143,7 +146,7 @@ func TestConstructPrepareMessage(test *testing.T) { ) consensus, err := New( - host, shard.BeaconChainShardID, multibls.GetPrivateKeys(blsPriKey1), nil, decider, 3, false, + host, shard.BeaconChainShardID, multibls.GetPrivateKeys(blsPriKey1), registry.New(), decider, 3, false, ) if err != nil { test.Fatalf("Cannot create consensus: %v", err) @@ -234,7 +237,7 @@ func TestConstructCommitMessage(test *testing.T) { quorum.SuperMajorityStake, shard.BeaconChainShardID, ) - consensus, err := New(host, shard.BeaconChainShardID, multibls.GetPrivateKeys(blsPriKey1), nil, decider, 3, false) + consensus, err := New(host, shard.BeaconChainShardID, multibls.GetPrivateKeys(blsPriKey1), registry.New(), decider, 3, false) if err != nil { test.Fatalf("Cannot create consensus: %v", err) } @@ -316,7 +319,7 @@ func TestPopulateMessageFields(t *testing.T) { quorum.SuperMajorityVote, shard.BeaconChainShardID, ) consensus, err := New( - host, shard.BeaconChainShardID, multibls.GetPrivateKeys(blsPriKey), nil, decider, 3, false, + host, shard.BeaconChainShardID, multibls.GetPrivateKeys(blsPriKey), registry.New(), decider, 3, false, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) diff --git a/consensus/downloader.go b/consensus/downloader.go index 8442ed5343..26755bbd25 100644 --- a/consensus/downloader.go +++ b/consensus/downloader.go @@ -114,7 +114,14 @@ func (consensus *Consensus) spinUpStateSync() { v.Stop() } } else { - consensus.spinLegacyStateSync() + select { + case consensus.BlockNumLowChan <- struct{}{}: + consensus.current.SetMode(Syncing) + for _, v := range consensus.consensusTimeout { + v.Stop() + } + default: + } } } diff --git a/consensus/leader.go b/consensus/leader.go index 2f7766e19b..1b10c96086 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -235,7 +235,7 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { Msg("[OnCommit] Failed finding a matching block for committed message") return } - commitPayload := signature.ConstructCommitPayload(consensus.Blockchain(), + commitPayload := signature.ConstructCommitPayload(consensus.Blockchain().Config(), blockObj.Epoch(), blockObj.Hash(), blockObj.NumberU64(), blockObj.Header().ViewID().Uint64()) logger = logger.With(). Uint64("MsgViewID", recvMsg.ViewID). diff --git a/consensus/signature/signature.go b/consensus/signature/signature.go index 3c419b03b0..20226a3811 100644 --- a/consensus/signature/signature.go +++ b/consensus/signature/signature.go @@ -8,18 +8,14 @@ import ( "github.com/harmony-one/harmony/internal/params" ) -type signatureChainReader interface { - Config() *params.ChainConfig -} - // ConstructCommitPayload returns the commit payload for consensus signatures. func ConstructCommitPayload( - chain signatureChainReader, epoch *big.Int, blockHash common.Hash, blockNum, viewID uint64, + config *params.ChainConfig, epoch *big.Int, blockHash common.Hash, blockNum, viewID uint64, ) []byte { blockNumBytes := make([]byte, 8) binary.LittleEndian.PutUint64(blockNumBytes, blockNum) commitPayload := append(blockNumBytes, blockHash.Bytes()...) - if !chain.Config().IsStaking(epoch) { + if !config.IsStaking(epoch) { return commitPayload } viewIDBytes := make([]byte, 8) diff --git a/consensus/threshold.go b/consensus/threshold.go index ecb54980a0..11e65709e6 100644 --- a/consensus/threshold.go +++ b/consensus/threshold.go @@ -46,7 +46,7 @@ func (consensus *Consensus) didReachPrepareQuorum() error { Msg("[didReachPrepareQuorum] Unparseable block data") return err } - commitPayload := signature.ConstructCommitPayload(consensus.Blockchain(), + commitPayload := signature.ConstructCommitPayload(consensus.Blockchain().Config(), blockObj.Epoch(), blockObj.Hash(), blockObj.NumberU64(), blockObj.Header().ViewID().Uint64()) // so by this point, everyone has committed to the blockhash of this block diff --git a/consensus/validator.go b/consensus/validator.go index 92008a91e6..c67765cc8a 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -164,7 +164,7 @@ func (consensus *Consensus) sendCommitMessages(blockObj *types.Block) { priKeys := consensus.getPriKeysInCommittee() // Sign commit signature on the received block and construct the p2p messages - commitPayload := signature.ConstructCommitPayload(consensus.Blockchain(), + commitPayload := signature.ConstructCommitPayload(consensus.Blockchain().Config(), blockObj.Epoch(), blockObj.Hash(), blockObj.NumberU64(), blockObj.Header().ViewID().Uint64()) p2pMsgs := consensus.constructP2pMessages(msg_pb.MessageType_COMMIT, commitPayload, priKeys) diff --git a/core/blockchain.go b/core/blockchain.go index 8afe622c70..facb9ed2c6 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -255,13 +255,6 @@ type BlockChain interface { ReadValidatorStats( addr common.Address, ) (*types2.ValidatorStats, error) - // UpdateValidatorVotingPower writes the voting power for the committees. - UpdateValidatorVotingPower( - batch rawdb.DatabaseWriter, - block *types.Block, - newEpochSuperCommittee, currentEpochSuperCommittee *shard.State, - state *state.DB, - ) (map[common.Address]*types2.ValidatorStats, error) // ComputeAndUpdateAPR ... ComputeAndUpdateAPR( block *types.Block, now *big.Int, diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index f1d0968422..4774b795fd 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -2654,8 +2654,8 @@ func (bc *BlockChainImpl) ReadValidatorStats( return rawdb.ReadValidatorStats(bc.db, addr) } -func (bc *BlockChainImpl) UpdateValidatorVotingPower( - batch rawdb.DatabaseWriter, +func UpdateValidatorVotingPower( + bc BlockChain, block *types.Block, newEpochSuperCommittee, currentEpochSuperCommittee *shard.State, state *state.DB, @@ -2681,7 +2681,7 @@ func (bc *BlockChainImpl) UpdateValidatorVotingPower( // bc.db, currentValidator, currentEpochSuperCommittee.Epoch, // ) // rawdb.DeleteValidatorStats(bc.db, currentValidator) - stats, err := rawdb.ReadValidatorStats(bc.db, currentValidator) + stats, err := rawdb.ReadValidatorStats(bc.ChainDb(), currentValidator) if err != nil { stats = staking.NewEmptyStats() } @@ -2731,7 +2731,7 @@ func (bc *BlockChainImpl) UpdateValidatorVotingPower( networkWide := votepower.AggregateRosters(rosters) for key, value := range networkWide { - stats, err := rawdb.ReadValidatorStats(bc.db, key) + stats, err := rawdb.ReadValidatorStats(bc.ChainDb(), key) if err != nil { stats = staking.NewEmptyStats() } diff --git a/core/offchain.go b/core/offchain.go index 3f08b4f81a..6ce7794e4f 100644 --- a/core/offchain.go +++ b/core/offchain.go @@ -205,8 +205,8 @@ func (bc *BlockChainImpl) CommitOffChainData( if shardState, err := shard.DecodeWrapper( header.ShardState(), ); err == nil { - if stats, err := bc.UpdateValidatorVotingPower( - batch, block, shardState, currentSuperCommittee, state, + if stats, err := UpdateValidatorVotingPower( + bc, block, shardState, currentSuperCommittee, state, ); err != nil { utils.Logger(). Err(err). diff --git a/internal/chain/engine.go b/internal/chain/engine.go index 6e018dccfd..71bb0d3052 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -632,7 +632,7 @@ func payloadArgsFromCrossLink(cl types.CrossLink) payloadArgs { } func (args payloadArgs) constructPayload(chain engine.ChainReader) []byte { - return signature.ConstructCommitPayload(chain, args.epoch, args.blockHash, args.number, args.viewID) + return signature.ConstructCommitPayload(chain.Config(), args.epoch, args.blockHash, args.number, args.viewID) } type sigArgs struct { diff --git a/internal/chain/engine_test.go b/internal/chain/engine_test.go index 5a842088c8..067b265ca0 100644 --- a/internal/chain/engine_test.go +++ b/internal/chain/engine_test.go @@ -275,7 +275,7 @@ func (kp blsKeyPair) Pub() bls.SerializedPublicKey { func (kp blsKeyPair) Sign(block *types.Block) []byte { chain := &fakeBlockChain{config: *params.LocalnetChainConfig} - msg := consensus_sig.ConstructCommitPayload(chain, block.Epoch(), block.Hash(), + msg := consensus_sig.ConstructCommitPayload(chain.Config(), block.Epoch(), block.Hash(), block.Number().Uint64(), block.Header().ViewID().Uint64()) sig := kp.pri.SignHash(msg) diff --git a/internal/registry/registry.go b/internal/registry/registry.go index 98b69a3ef5..448e148668 100644 --- a/internal/registry/registry.go +++ b/internal/registry/registry.go @@ -4,13 +4,16 @@ import ( "sync" "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/webhooks" ) // Registry consolidates services at one place. type Registry struct { - mu sync.Mutex - blockchain core.BlockChain - txPool *core.TxPool + mu sync.Mutex + blockchain core.BlockChain + beaconchain core.BlockChain + webHooks *webhooks.Hooks + txPool *core.TxPool } // New creates a new registry. @@ -35,6 +38,40 @@ func (r *Registry) GetBlockchain() core.BlockChain { return r.blockchain } +// SetBeaconchain sets the beaconchain to registry. +func (r *Registry) SetBeaconchain(bc core.BlockChain) *Registry { + r.mu.Lock() + defer r.mu.Unlock() + + r.beaconchain = bc + return r +} + +// GetBeaconchain gets the beaconchain from registry. +func (r *Registry) GetBeaconchain() core.BlockChain { + r.mu.Lock() + defer r.mu.Unlock() + + return r.beaconchain +} + +// SetWebHooks sets the webhooks to registry. +func (r *Registry) SetWebHooks(hooks *webhooks.Hooks) *Registry { + r.mu.Lock() + defer r.mu.Unlock() + + r.webHooks = hooks + return r +} + +// GetWebHooks gets the webhooks from registry. +func (r *Registry) GetWebHooks() *webhooks.Hooks { + r.mu.Lock() + defer r.mu.Unlock() + + return r.webHooks +} + // SetTxPool sets the txpool to registry. func (r *Registry) SetTxPool(txPool *core.TxPool) *Registry { r.mu.Lock() diff --git a/node/node_explorer.go b/node/node_explorer.go index d1c78d612a..0dd7319761 100644 --- a/node/node_explorer.go +++ b/node/node_explorer.go @@ -68,7 +68,7 @@ func (node *Node) explorerMessageHandler(ctx context.Context, msg *msg_pb.Messag return errBlockBeforeCommit } - commitPayload := signature.ConstructCommitPayload(node.Blockchain(), + commitPayload := signature.ConstructCommitPayload(node.Blockchain().Config(), block.Epoch(), block.Hash(), block.Number().Uint64(), block.Header().ViewID().Uint64()) if !aggSig.VerifyHash(mask.AggregatePublic, commitPayload) { utils.Logger(). diff --git a/node/node_handler.go b/node/node_handler.go index acc4afc266..a7e496e98a 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -25,7 +25,6 @@ import ( "github.com/harmony-one/harmony/staking/slash" staking "github.com/harmony-one/harmony/staking/types" "github.com/harmony-one/harmony/webhooks" - "github.com/pkg/errors" ) const p2pMsgPrefixSize = 5 @@ -328,38 +327,6 @@ func getCrosslinkHeadersForShards(shardChain core.BlockChain, curBlock *types.Bl return headers, nil } -// VerifyNewBlock is called by consensus participants to verify the block (account model) they are -// running consensus on. -func VerifyNewBlock(nodeConfig *nodeconfig.ConfigType, blockChain core.BlockChain, beaconChain core.BlockChain) func(*types.Block) error { - return func(newBlock *types.Block) error { - if err := blockChain.ValidateNewBlock(newBlock, beaconChain); err != nil { - if hooks := nodeConfig.WebHooks.Hooks; hooks != nil { - if p := hooks.ProtocolIssues; p != nil { - url := p.OnCannotCommit - go func() { - webhooks.DoPost(url, map[string]interface{}{ - "bad-header": newBlock.Header(), - "reason": err.Error(), - }) - }() - } - } - utils.Logger().Error(). - Str("blockHash", newBlock.Hash().Hex()). - Int("numTx", len(newBlock.Transactions())). - Int("numStakingTx", len(newBlock.StakingTransactions())). - Err(err). - Msg("[VerifyNewBlock] Cannot Verify New Block!!!") - return errors.Errorf( - "[VerifyNewBlock] Cannot Verify New Block!!! block-hash %s txn-count %d", - newBlock.Hash().Hex(), - len(newBlock.Transactions()), - ) - } - return nil - } -} - // PostConsensusProcessing is called by consensus participants, after consensus is done, to: // 1. [leader] send new block to the client // 2. [leader] send cross shard tx receipts to destination shard @@ -373,6 +340,7 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block) error { node.BroadcastCXReceipts(newBlock) } else { if node.Consensus.Mode() != consensus.Listening { + numSignatures := node.Consensus.NumSignaturesIncludedInBlock(newBlock) utils.Logger().Info(). Uint64("blockNum", newBlock.NumberU64()). Uint64("epochNum", newBlock.Epoch().Uint64()). @@ -380,11 +348,10 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block) error { Str("blockHash", newBlock.Hash().String()). Int("numTxns", len(newBlock.Transactions())). Int("numStakingTxns", len(newBlock.StakingTransactions())). - Uint32("numSignatures", node.Consensus.NumSignaturesIncludedInBlock(newBlock)). + Uint32("numSignatures", numSignatures). Msg("BINGO !!! Reached Consensus") - numSig := float64(node.Consensus.NumSignaturesIncludedInBlock(newBlock)) - node.Consensus.UpdateValidatorMetrics(numSig, float64(newBlock.NumberU64())) + node.Consensus.UpdateValidatorMetrics(float64(numSignatures), float64(newBlock.NumberU64())) // 1% of the validator also need to do broadcasting rnd := rand.Intn(100) diff --git a/node/node_handler_test.go b/node/node_handler_test.go index 25d966f469..3de2918398 100644 --- a/node/node_handler_test.go +++ b/node/node_handler_test.go @@ -103,7 +103,7 @@ func TestVerifyNewBlock(t *testing.T) { t.Fatal("cannot get blockchain") } reg := registry.New().SetBlockchain(blockchain) - consensus, err := consensus.New( + consensusObj, err := consensus.New( host, shard.BeaconChainShardID, multibls.GetPrivateKeys(blsKey), reg, decider, 3, false, ) if err != nil { @@ -112,7 +112,7 @@ func TestVerifyNewBlock(t *testing.T) { archiveMode := make(map[uint32]bool) archiveMode[0] = true archiveMode[1] = false - node := New(host, consensus, engine, collection, nil, nil, nil, archiveMode, nil, reg) + node := New(host, consensusObj, engine, collection, nil, nil, nil, archiveMode, nil, reg) txs := make(map[common.Address]types.Transactions) stks := staking.StakingTransactions{} @@ -129,7 +129,7 @@ func TestVerifyNewBlock(t *testing.T) { // work around vrf verification as it's tested in another test. node.Blockchain().Config().VRFEpoch = big.NewInt(2) - if err := VerifyNewBlock(nil, node.Blockchain(), node.Beaconchain())(block); err != nil { + if err := consensus.VerifyNewBlock(nil, node.Blockchain(), node.Beaconchain())(block); err != nil { t.Error("New block is not verified successfully:", err) } } diff --git a/node/node_newblock_test.go b/node/node_newblock_test.go index 492175b1d2..39affacaba 100644 --- a/node/node_newblock_test.go +++ b/node/node_newblock_test.go @@ -46,14 +46,15 @@ func TestFinalizeNewBlockAsync(t *testing.T) { decider := quorum.NewDecider( quorum.SuperMajorityVote, shard.BeaconChainShardID, ) - consensus, err := consensus.New( - host, shard.BeaconChainShardID, multibls.GetPrivateKeys(blsKey), nil, decider, 3, false, + reg := registry.New().SetBlockchain(blockchain).SetBeaconchain(blockchain) + consensusObj, err := consensus.New( + host, shard.BeaconChainShardID, multibls.GetPrivateKeys(blsKey), reg, decider, 3, false, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) } - node := New(host, consensus, engine, collection, nil, nil, nil, nil, nil, registry.New().SetBlockchain(blockchain)) + node := New(host, consensusObj, engine, collection, nil, nil, nil, nil, nil, registry.New().SetBlockchain(blockchain)) node.Worker.UpdateCurrent() @@ -71,7 +72,7 @@ func TestFinalizeNewBlockAsync(t *testing.T) { commitSigs, func() uint64 { return 0 }, common.Address{}, nil, nil, ) - if err := VerifyNewBlock(nil, blockchain, nil)(block); err != nil { + if err := consensus.VerifyNewBlock(nil, blockchain, nil)(block); err != nil { t.Error("New block is not verified successfully:", err) } diff --git a/staking/slash/double-sign.go b/staking/slash/double-sign.go index 6c871f9920..aaaea01300 100644 --- a/staking/slash/double-sign.go +++ b/staking/slash/double-sign.go @@ -248,7 +248,7 @@ func Verify( publicKey.Add(publicKeyObj) } // slash verification only happens in staking era, therefore want commit payload for staking epoch - commitPayload := consensus_sig.ConstructCommitPayload(chain, + commitPayload := consensus_sig.ConstructCommitPayload(chain.Config(), candidate.Evidence.Epoch, ballot.BlockHeaderHash, candidate.Evidence.Height, candidate.Evidence.ViewID) utils.Logger().Debug(). Uint64("epoch", candidate.Evidence.Epoch.Uint64()). diff --git a/staking/slash/double-sign_test.go b/staking/slash/double-sign_test.go index 470a8ae1d8..7bed53b94e 100644 --- a/staking/slash/double-sign_test.go +++ b/staking/slash/double-sign_test.go @@ -1151,8 +1151,7 @@ func (kp blsKeyPair) Pub() bls.SerializedPublicKey { } func (kp blsKeyPair) Sign(block *types.Block) []byte { - chain := &fakeBlockChain{config: *params.LocalnetChainConfig} - msg := consensus_sig.ConstructCommitPayload(chain, block.Epoch(), block.Hash(), + msg := consensus_sig.ConstructCommitPayload(params.LocalnetChainConfig, block.Epoch(), block.Hash(), block.Number().Uint64(), block.Header().ViewID().Uint64()) sig := kp.pri.SignHash(msg) diff --git a/staking/verify/verify.go b/staking/verify/verify.go index 38783d4dbe..efc7e18ce7 100644 --- a/staking/verify/verify.go +++ b/staking/verify/verify.go @@ -1,54 +1 @@ package verify - -import ( - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/harmony-one/bls/ffi/go/bls" - "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/consensus/signature" - "github.com/harmony-one/harmony/core" - bls_cosi "github.com/harmony-one/harmony/crypto/bls" - "github.com/harmony-one/harmony/shard" - "github.com/pkg/errors" -) - -var ( - errQuorumVerifyAggSign = errors.New("insufficient voting power to verify aggregate sig") - errAggregateSigFail = errors.New("could not verify hash of aggregate signature") -) - -// AggregateSigForCommittee .. -func AggregateSigForCommittee( - chain core.BlockChain, - committee *shard.Committee, - decider quorum.Decider, - aggSignature *bls.Sign, - hash common.Hash, - blockNum, viewID uint64, - epoch *big.Int, - bitmap []byte, -) error { - committerKeys, err := committee.BLSPublicKeys() - if err != nil { - return err - } - mask, err := bls_cosi.NewMask(committerKeys, nil) - if err != nil { - return err - } - if err := mask.SetMask(bitmap); err != nil { - return err - } - - if !decider.IsQuorumAchievedByMask(mask) { - return errQuorumVerifyAggSign - } - - commitPayload := signature.ConstructCommitPayload(chain, epoch, hash, blockNum, viewID) - if !aggSignature.VerifyHash(mask.AggregatePublic, commitPayload) { - return errAggregateSigFail - } - - return nil -} From 10119bd8d51e56bd65605c64f0ab38469c27f062 Mon Sep 17 00:00:00 2001 From: Konstantin <355847+Frozen@users.noreply.github.com> Date: Mon, 5 Jun 2023 15:13:56 -0400 Subject: [PATCH 411/420] Fixed race errors in downloader & consensus (#4442) * Fix race. * Fix race. * Added log. * Fix panic. --- consensus/consensus.go | 2 ++ consensus/debug.go | 2 ++ p2p/stream/common/streammanager/streammanager.go | 10 +++++++++- p2p/stream/protocols/sync/protocol.go | 2 +- p2p/stream/protocols/sync/stream.go | 2 +- 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index ca2890fbbb..42d57047c0 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -188,6 +188,8 @@ func (consensus *Consensus) BlocksSynchronized() { // BlocksNotSynchronized lets the main loop know that block is not synchronized func (consensus *Consensus) BlocksNotSynchronized() { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() consensus.syncNotReadyChan() } diff --git a/consensus/debug.go b/consensus/debug.go index cba13cc011..a323e04ed6 100644 --- a/consensus/debug.go +++ b/consensus/debug.go @@ -14,6 +14,8 @@ func (consensus *Consensus) getConsensusPhase() string { // GetConsensusMode returns the current mode of the consensus func (consensus *Consensus) GetConsensusMode() string { + consensus.mutex.RLock() + defer consensus.mutex.RUnlock() return consensus.current.mode.String() } diff --git a/p2p/stream/common/streammanager/streammanager.go b/p2p/stream/common/streammanager/streammanager.go index e57919a76f..26025fb720 100644 --- a/p2p/stream/common/streammanager/streammanager.go +++ b/p2p/stream/common/streammanager/streammanager.go @@ -299,7 +299,7 @@ func (sm *streamManager) removeAllStreamOnClose() { wg.Wait() // Be nice. after close, the field is still accessible to prevent potential panics - sm.streams = newStreamSet() + sm.streams.Erase() } func (sm *streamManager) discoverAndSetupStream(discCtx context.Context) (int, error) { @@ -403,6 +403,14 @@ func newStreamSet() *streamSet { } } +func (ss *streamSet) Erase() { + ss.lock.Lock() + defer ss.lock.Unlock() + + ss.streams = make(map[sttypes.StreamID]sttypes.Stream) + ss.numByProto = make(map[sttypes.ProtoSpec]int) +} + func (ss *streamSet) size() int { ss.lock.RLock() defer ss.lock.RUnlock() diff --git a/p2p/stream/protocols/sync/protocol.go b/p2p/stream/protocols/sync/protocol.go index 80ea0927bc..3dcbd6f063 100644 --- a/p2p/stream/protocols/sync/protocol.go +++ b/p2p/stream/protocols/sync/protocol.go @@ -180,7 +180,7 @@ func (p *Protocol) HandleStream(raw libp2p_network.Stream) { Msg("failed to add new stream") return } - fmt.Println("Connected to", raw.Conn().RemotePeer().String(), "(", st.ProtoID(), ")") + fmt.Println("Connected to", raw.Conn().RemotePeer().String(), "(", st.ProtoID(), ")", "my ID: ", raw.Conn().LocalPeer().String()) st.run() } diff --git a/p2p/stream/protocols/sync/stream.go b/p2p/stream/protocols/sync/stream.go index 0af4fff7c4..467588deaf 100644 --- a/p2p/stream/protocols/sync/stream.go +++ b/p2p/stream/protocols/sync/stream.go @@ -298,7 +298,7 @@ func (st *syncStream) computeBlockNumberResp(rid uint64) *syncpb.Message { return syncpb.MakeGetBlockNumberResponseMessage(rid, bn) } -func (st syncStream) computeGetBlockHashesResp(rid uint64, bns []uint64) (*syncpb.Message, error) { +func (st *syncStream) computeGetBlockHashesResp(rid uint64, bns []uint64) (*syncpb.Message, error) { if len(bns) > GetBlockHashesAmountCap { err := fmt.Errorf("GetBlockHashes amount exceed cap: %v>%v", len(bns), GetBlockHashesAmountCap) return nil, err From 6a2f9cd4c7f9a32a60cf3d9dcd5c76f393913425 Mon Sep 17 00:00:00 2001 From: Konstantin <355847+Frozen@users.noreply.github.com> Date: Tue, 6 Jun 2023 12:47:05 -0400 Subject: [PATCH 412/420] Fix panic. (#4440) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix panic. * check for empty blockBytes in stage bodies --------- Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> --- api/service/stagedstreamsync/block_manager.go | 6 +++++- api/service/stagedstreamsync/stage_bodies.go | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/api/service/stagedstreamsync/block_manager.go b/api/service/stagedstreamsync/block_manager.go index df30ab5e36..28c966b4d6 100644 --- a/api/service/stagedstreamsync/block_manager.go +++ b/api/service/stagedstreamsync/block_manager.go @@ -84,7 +84,7 @@ func (gbm *blockDownloadManager) HandleRequestResult(bns []uint64, blockBytes [] for i, bn := range bns { delete(gbm.requesting, bn) - if len(blockBytes[i]) <= 1 { + if indexExists(blockBytes, i) && len(blockBytes[i]) <= 1 { gbm.retries.push(bn) } else { gbm.processing[bn] = struct{}{} @@ -97,6 +97,10 @@ func (gbm *blockDownloadManager) HandleRequestResult(bns []uint64, blockBytes [] return nil } +func indexExists[T any](slice []T, index int) bool { + return index >= 0 && index < len(slice) +} + // SetDownloadDetails sets the download details for a batch of blocks func (gbm *blockDownloadManager) SetDownloadDetails(bns []uint64, loopID int, streamID sttypes.StreamID) error { gbm.lock.Lock() diff --git a/api/service/stagedstreamsync/stage_bodies.go b/api/service/stagedstreamsync/stage_bodies.go index 1fbfcbc2e6..cb06e98e8a 100644 --- a/api/service/stagedstreamsync/stage_bodies.go +++ b/api/service/stagedstreamsync/stage_bodies.go @@ -168,6 +168,13 @@ func (b *StageBodies) runBlockWorkerLoop(gbm *blockDownloadManager, wg *sync.Wai Msg(WrapStagedSyncMsg("downloadRawBlocks failed")) err = errors.Wrap(err, "request error") gbm.HandleRequestError(batch, err, stid) + } else if blockBytes == nil || len(blockBytes) == 0 { + utils.Logger().Warn(). + Str("stream", string(stid)). + Interface("block numbers", batch). + Msg(WrapStagedSyncMsg("downloadRawBlocks failed, received empty blockBytes")) + err := errors.New("downloadRawBlocks received empty blockBytes") + gbm.HandleRequestError(batch, err, stid) } else { if err = b.saveBlocks(gbm.tx, batch, blockBytes, sigBytes, loopID, stid); err != nil { panic(ErrSaveBlocksToDbFailed) From 61b8aba82c042680d1cfb6b6135ec0e670dac633 Mon Sep 17 00:00:00 2001 From: Gheis Mohammadi Date: Wed, 7 Jun 2023 01:14:45 +0800 Subject: [PATCH 413/420] Snapshot integration and add cache to statedb (#4419) * integrate snapshot feature with statedb and add cache to states * send nil as stateCache to let NewDatabase function decides about cache configs * fix test issues of using snapshot --- core/blockchain_impl.go | 116 +++++++++++++++++++++++++---- core/evm_test.go | 3 +- core/state/database.go | 5 +- core/state/snapshot/snapshot.go | 2 +- core/state/statedb.go | 2 +- core/tx_pool_test.go | 3 +- internal/shardchain/shardchains.go | 5 +- node/worker/worker_test.go | 8 +- test/chain/main.go | 3 +- test/chain/reward/main.go | 2 +- 10 files changed, 121 insertions(+), 28 deletions(-) diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 4774b795fd..2c9e306205 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -39,6 +39,7 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" bls2 "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/block" consensus_engine "github.com/harmony-one/harmony/consensus/engine" @@ -46,6 +47,7 @@ import ( "github.com/harmony-one/harmony/consensus/votepower" "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/state" + "github.com/harmony-one/harmony/core/state/snapshot" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/crypto/bls" @@ -124,10 +126,29 @@ const ( // CacheConfig contains the configuration values for the trie caching/pruning // that's resident in a blockchain. type CacheConfig struct { - Disabled bool // Whether to disable trie write caching (archive node) - TrieNodeLimit int // Memory limit (MB) at which to flush the current in-memory trie to disk - TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk - TriesInMemory uint64 // Block number from the head stored in disk before exiting + Disabled bool // Whether to disable trie write caching (archive node) + TrieNodeLimit int // Memory limit (MB) at which to flush the current in-memory trie to disk + TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk + TriesInMemory uint64 // Block number from the head stored in disk before exiting + TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk + TrieDirtyDisabled bool // Whether to disable trie write caching and GC altogether (archive node) + TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory + TrieCleanJournal string // Disk journal for saving clean cache entries. + Preimages bool // Whether to store preimage of trie key to the disk + SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory + SnapshotNoBuild bool // Whether the background generation is allowed + SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it +} + +// defaultCacheConfig are the default caching values if none are specified by the +// user (also used during testing). +var defaultCacheConfig = &CacheConfig{ + Disabled: false, + TrieCleanLimit: 256, + TrieDirtyLimit: 256, + TrieTimeLimit: 5 * time.Minute, + SnapshotLimit: 256, + SnapshotWait: true, } type BlockChainImpl struct { @@ -137,8 +158,10 @@ type BlockChainImpl struct { shardID uint32 // Shard number db ethdb.Database // Low level persistent database to store final content in + snaps *snapshot.Tree // Snapshot tree for fast trie leaf access triegc *prque.Prque[int64, common.Hash] // Priority queue mapping block numbers to tries to gc gcproc time.Duration // Accumulates canonical block processing for trie dumping + triedb *trie.Database // The database handler for maintaining trie nodes. // The following two variables are used to clean up the cache of redis in tikv mode. // This can improve the cache hit rate of redis @@ -225,6 +248,21 @@ func newBlockChainWithOptions( cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus_engine.Engine, vmConfig vm.Config, options Options) (*BlockChainImpl, error) { + if cacheConfig == nil { + cacheConfig = defaultCacheConfig + } + + // Open trie database with provided config + triedb := trie.NewDatabaseWithConfig(db, &trie.Config{ + Cache: cacheConfig.TrieCleanLimit, + Journal: cacheConfig.TrieCleanJournal, + Preimages: cacheConfig.Preimages, + }) + + if stateCache == nil { + stateCache = state.NewDatabaseWithNodeDB(db, triedb) + } + bodyCache, _ := lru.New(bodyCacheLimit) bodyRLPCache, _ := lru.New(bodyCacheLimit) receiptsCache, _ := lru.New(receiptsCacheLimit) @@ -248,6 +286,7 @@ func newBlockChainWithOptions( cacheConfig: cacheConfig, db: db, triegc: prque.New[int64, common.Hash](nil), + triedb: triedb, stateCache: stateCache, quit: make(chan struct{}), bodyCache: bodyCache, @@ -298,6 +337,28 @@ func newBlockChainWithOptions( bc.SetValidator(NewBlockValidator(chainConfig, bc, engine)) bc.SetProcessor(NewStateProcessor(chainConfig, bc, beaconChain, engine)) + // Load any existing snapshot, regenerating it if loading failed + if bc.cacheConfig.SnapshotLimit > 0 { + // If the chain was rewound past the snapshot persistent layer (causing + // a recovery block number to be persisted to disk), check if we're still + // in recovery mode and in that case, don't invalidate the snapshot on a + // head mismatch. + var recover bool + + head := bc.CurrentBlock() + if layer := rawdb.ReadSnapshotRecoveryNumber(bc.db); layer != nil && *layer >= head.NumberU64() { + utils.Logger().Warn().Uint64("diskbase", *layer).Uint64("chainhead", head.NumberU64()).Msg("Enabling snapshot recovery") + recover = true + } + snapconfig := snapshot.Config{ + CacheSize: bc.cacheConfig.SnapshotLimit, + Recovery: recover, + NoBuild: bc.cacheConfig.SnapshotNoBuild, + AsyncBuild: !bc.cacheConfig.SnapshotWait, + } + bc.snaps, _ = snapshot.New(snapconfig, bc.db, bc.triedb, head.Hash()) + } + // Take ownership of this particular state go bc.update() return bc, nil @@ -463,7 +524,7 @@ func (bc *BlockChainImpl) ValidateNewBlock(block *types.Block, beaconChain Block } func (bc *BlockChainImpl) validateNewBlock(block *types.Block) error { - state, err := state.New(bc.CurrentBlock().Root(), bc.stateCache, nil) + state, err := state.New(bc.CurrentBlock().Root(), bc.stateCache, bc.snaps) if err != nil { return err } @@ -521,7 +582,7 @@ func (bc *BlockChainImpl) loadLastState() error { return bc.Reset() } // Make sure the state associated with the block is available - if _, err := state.New(currentBlock.Root(), bc.stateCache, nil); err != nil { + if _, err := state.New(currentBlock.Root(), bc.stateCache, bc.snaps); err != nil { // Dangling block without a state associated, init from scratch utils.Logger().Warn(). Str("number", currentBlock.Number().String()). @@ -618,7 +679,7 @@ func (bc *BlockChainImpl) SetHead(head uint64) error { headBlockGauge.Update(int64(newHeadBlock.NumberU64())) } if currentBlock := bc.CurrentBlock(); currentBlock != nil { - if _, err := state.New(currentBlock.Root(), bc.stateCache, nil); err != nil { + if _, err := state.New(currentBlock.Root(), bc.stateCache, bc.snaps); err != nil { // Rewound state missing, rolled back to before pivot, reset to genesis bc.currentBlock.Store(bc.genesisBlock) headBlockGauge.Update(int64(bc.genesisBlock.NumberU64())) @@ -695,7 +756,12 @@ func (bc *BlockChainImpl) State() (*state.DB, error) { } func (bc *BlockChainImpl) StateAt(root common.Hash) (*state.DB, error) { - return state.New(root, bc.stateCache, nil) + return state.New(root, bc.stateCache, bc.snaps) +} + +// Snapshots returns the blockchain snapshot tree. +func (bc *BlockChainImpl) Snapshots() *snapshot.Tree { + return bc.snaps } func (bc *BlockChainImpl) Reset() error { @@ -740,7 +806,7 @@ func (bc *BlockChainImpl) repair(head **types.Block) error { valsToRemove := map[common.Address]struct{}{} for { // Abort if we've rewound to a head block that does have associated state - if _, err := state.New((*head).Root(), bc.stateCache, nil); err == nil { + if _, err := state.New((*head).Root(), bc.stateCache, bc.snaps); err == nil { utils.Logger().Info(). Str("number", (*head).Number().String()). Str("hash", (*head).Hash().Hex()). @@ -1052,6 +1118,15 @@ func (bc *BlockChainImpl) Stop() { return } + // Ensure that the entirety of the state snapshot is journalled to disk. + var snapBase common.Hash + if bc.snaps != nil { + var err error + if snapBase, err = bc.snaps.Journal(bc.CurrentBlock().Header().Root()); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to journal state snapshot") + } + } + if !atomic.CompareAndSwapInt32(&bc.running, 0, 1) { return } @@ -1091,6 +1166,12 @@ func (bc *BlockChainImpl) Stop() { } } } + if snapBase != (common.Hash{}) { + utils.Logger().Info().Interface("root", snapBase).Msg("Writing snapshot state to disk") + if err := triedb.Commit(snapBase, true); err != nil { + utils.Logger().Error().Err(err).Msg("Failed to commit recent state trie") + } + } for !bc.triegc.Empty() { v := common.Hash(bc.triegc.PopItem()) triedb.Dereference(v) @@ -1099,6 +1180,15 @@ func (bc *BlockChainImpl) Stop() { utils.Logger().Error().Msg("Dangling trie nodes after full cleanup") } } + // Flush the collected preimages to disk + if err := bc.stateCache.TrieDB().CommitPreimages(); err != nil { + utils.Logger().Error().Interface("err", err).Msg("Failed to commit trie preimages") + } + // Ensure all live cached entries be saved into disk, so that we can skip + // cache warmup when node restarts. + if bc.cacheConfig.TrieCleanJournal != "" { + bc.triedb.SaveCache(bc.cacheConfig.TrieCleanJournal) + } utils.Logger().Info().Msg("Blockchain manager stopped") } @@ -1733,7 +1823,7 @@ func (bc *BlockChainImpl) insertChain(chain types.Blocks, verifyHeaders bool) (i } else { parent = chain[i-1] } - state, err := state.New(parent.Root(), bc.stateCache, nil) + state, err := state.New(parent.Root(), bc.stateCache, bc.snaps) if err != nil { return i, events, coalescedLogs, err } @@ -3418,14 +3508,14 @@ func (bc *BlockChainImpl) tikvCleanCache() { for i := bc.latestCleanCacheNum + 1; i <= to; i++ { // build previous block statedb fromBlock := bc.GetBlockByNumber(i) - fromTrie, err := state.New(fromBlock.Root(), bc.stateCache, nil) + fromTrie, err := state.New(fromBlock.Root(), bc.stateCache, bc.snaps) if err != nil { continue } // build current block statedb toBlock := bc.GetBlockByNumber(i + 1) - toTrie, err := state.New(toBlock.Root(), bc.stateCache, nil) + toTrie, err := state.New(toBlock.Root(), bc.stateCache, bc.snaps) if err != nil { continue } @@ -3525,7 +3615,7 @@ func (bc *BlockChainImpl) InitTiKV(conf *harmonyconfig.TiKVConfig) { // If redis is empty, the hit rate will be too low and the synchronization block speed will be slow // set LOAD_PRE_FETCH is yes can significantly improve this. if os.Getenv("LOAD_PRE_FETCH") == "yes" { - if trie, err := state.New(bc.CurrentBlock().Root(), bc.stateCache, nil); err == nil { + if trie, err := state.New(bc.CurrentBlock().Root(), bc.stateCache, bc.snaps); err == nil { trie.Prefetch(512) } else { log.Println("LOAD_PRE_FETCH ERR: ", err) diff --git a/core/evm_test.go b/core/evm_test.go index cab3f712cf..04399ff14c 100644 --- a/core/evm_test.go +++ b/core/evm_test.go @@ -46,7 +46,8 @@ func getTestEnvironment(testBankKey ecdsa.PrivateKey) (*BlockChainImpl, *state.D genesis := gspec.MustCommit(database) // fake blockchain - chain, _ := NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, engine, vm.Config{}) + cacheConfig := &CacheConfig{SnapshotLimit: 0} + chain, _ := NewBlockChain(database, nil, nil, cacheConfig, gspec.Config, engine, vm.Config{}) db, _ := chain.StateAt(genesis.Root()) // make a fake block header (use epoch 1 so that locked tokens can be tested) diff --git a/core/state/database.go b/core/state/database.go index c1e375ccde..a563678376 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -139,7 +139,10 @@ func NewDatabase(db ethdb.Database) Database { } func NewDatabaseWithCache(db ethdb.Database, cache int) Database { - return NewDatabaseWithConfig(db, nil) + config := trie.Config{ + Cache: cache, + } + return NewDatabaseWithConfig(db, &config) } // NewDatabaseWithConfig creates a backing store for state. The returned database diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 47dc1a3a17..210fcac62b 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -206,7 +206,7 @@ func New(config Config, diskdb ethdb.KeyValueStore, triedb *trie.Database, root utils.Logger().Warn().Err(err).Msg("Snapshot maintenance disabled (syncing)") return snap, nil } - // Create the building waiter iff the background generation is allowed + // Create the building waiter if the background generation is allowed if !config.NoBuild && !config.AsyncBuild { defer snap.waitBuild() } diff --git a/core/state/statedb.go b/core/state/statedb.go index 9c1eefde82..e84084c02c 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -24,13 +24,13 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/harmony-one/harmony/core/rawdb" + "github.com/harmony-one/harmony/core/state/snapshot" types2 "github.com/harmony-one/harmony/core/types" common2 "github.com/harmony-one/harmony/internal/common" diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 9b126eac90..4e1c214b1a 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -161,7 +161,8 @@ func createBlockChain() *BlockChainImpl { genesis := gspec.MustCommit(database) _ = genesis engine := chain2.NewEngine() - blockchain, _ := NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, engine, vm.Config{}) + cacheConfig := &CacheConfig{SnapshotLimit: 0} + blockchain, _ := NewBlockChain(database, nil, nil, cacheConfig, gspec.Config, engine, vm.Config{}) return blockchain } diff --git a/internal/shardchain/shardchains.go b/internal/shardchain/shardchains.go index 1be2c68414..66cfad220a 100644 --- a/internal/shardchain/shardchains.go +++ b/internal/shardchain/shardchains.go @@ -107,7 +107,7 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c Msg("disable cache, running in archival mode") } else { cacheConfig = &core.CacheConfig{ - TrieNodeLimit: 256 * 1024 * 1024, + TrieNodeLimit: 256, TrieTimeLimit: 2 * time.Minute, TriesInMemory: 128, } @@ -172,9 +172,8 @@ func initStateCache(db ethdb.Database, sc *CollectionImpl, shardID uint32) (stat return nil, err } return state.NewDatabaseWithCache(stateDB, 64), nil - } else { - return state.NewDatabase(db), nil } + return nil, nil } // DisableCache disables caching mode for newly opened chains. diff --git a/node/worker/worker_test.go b/node/worker/worker_test.go index 37d61816cc..4c87c99377 100644 --- a/node/worker/worker_test.go +++ b/node/worker/worker_test.go @@ -5,8 +5,6 @@ import ( "math/rand" "testing" - "github.com/harmony-one/harmony/core/state" - "github.com/harmony-one/harmony/core/rawdb" "github.com/ethereum/go-ethereum/common" @@ -45,7 +43,8 @@ func TestNewWorker(t *testing.T) { genesis := gspec.MustCommit(database) _ = genesis - chain, err := core.NewBlockChain(database, state.NewDatabase(database), &core.BlockChainImpl{}, nil, gspec.Config, engine, vm.Config{}) + cacheConfig := &core.CacheConfig{SnapshotLimit: 0} + chain, err := core.NewBlockChain(database, nil, &core.BlockChainImpl{}, cacheConfig, gspec.Config, engine, vm.Config{}) if err != nil { t.Error(err) @@ -72,7 +71,8 @@ func TestCommitTransactions(t *testing.T) { ) gspec.MustCommit(database) - chain, _ := core.NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, engine, vm.Config{}) + cacheConfig := &core.CacheConfig{SnapshotLimit: 0} + chain, _ := core.NewBlockChain(database, nil, nil, cacheConfig, gspec.Config, engine, vm.Config{}) // Create a new worker worker := New(params.TestChainConfig, chain, nil, engine) diff --git a/test/chain/main.go b/test/chain/main.go index 656e90c949..ec28ead0cd 100644 --- a/test/chain/main.go +++ b/test/chain/main.go @@ -15,7 +15,6 @@ import ( blockfactory "github.com/harmony-one/harmony/block/factory" "github.com/harmony-one/harmony/core" core_state "github.com/harmony-one/harmony/core/state" - harmonyState "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/core/vm" "github.com/harmony-one/harmony/crypto/hash" @@ -206,7 +205,7 @@ func playFaucetContract(chain core.BlockChain) { func main() { genesis := gspec.MustCommit(database) - chain, _ := core.NewBlockChain(database, harmonyState.NewDatabase(database), nil, nil, gspec.Config, chain.Engine(), vm.Config{}) + chain, _ := core.NewBlockChain(database, nil, nil, nil, gspec.Config, chain.Engine(), vm.Config{}) txpool := core.NewTxPool(core.DefaultTxPoolConfig, chainConfig, chain, types.NewTransactionErrorSink()) backend := &testWorkerBackend{ diff --git a/test/chain/reward/main.go b/test/chain/reward/main.go index 5cfe9425a1..17be3b2715 100644 --- a/test/chain/reward/main.go +++ b/test/chain/reward/main.go @@ -109,7 +109,7 @@ func main() { genesis := gspec.MustCommit(database) _ = genesis engine := chain.NewEngine() - bc, _ := core.NewBlockChain(database, state.NewDatabase(database), nil, nil, gspec.Config, engine, vm.Config{}) + bc, _ := core.NewBlockChain(database, nil, nil, nil, gspec.Config, engine, vm.Config{}) statedb, _ := state.New(common2.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) msg := createValidator() statedb.AddBalance(msg.ValidatorAddress, new(big.Int).Mul(big.NewInt(5e18), big.NewInt(2000))) From 5cef0486dfb507ab0875da0377533b7a9b63303b Mon Sep 17 00:00:00 2001 From: Konstantin <355847+Frozen@users.noreply.github.com> Date: Wed, 7 Jun 2023 13:42:48 -0400 Subject: [PATCH 414/420] Consensus: removed double initialization. (#4404) * Removed hardcoded blockchain. * Fix for consensus. * Fix tests. * Fixed panic. * Clean up code. * Fix. * Removed Unused methods. * Set is backup. * Provide consensus to BroadcastCXReceipts. * Registry for CxPool. * CxPool & use Header instead of Block. * Fix conflicts. --- block/header.go | 5 ++++ cmd/harmony/main.go | 19 +++++++++++--- consensus/consensus.go | 28 ++++++++++----------- consensus/view_change_construct.go | 5 ---- core/blockchain.go | 2 +- core/blockchain_impl.go | 6 ++--- core/blockchain_stub.go | 2 +- internal/registry/registry.go | 34 +++++++++++++++++++++++++ node/node.go | 3 ++- node/node_cross_shard.go | 40 +++++++++++++++++------------- node/node_handler.go | 14 +++++------ node/node_newblock.go | 2 +- 12 files changed, 105 insertions(+), 55 deletions(-) diff --git a/block/header.go b/block/header.go index 5672446cc7..9b20535a60 100644 --- a/block/header.go +++ b/block/header.go @@ -122,6 +122,11 @@ func (h *Header) Hash() common.Hash { return hash.FromRLP(h) } +// NumberU64 returns the block number of the header as a uint64. +func (h *Header) NumberU64() uint64 { + return h.Number().Uint64() +} + // Logger returns a sub-logger with block contexts added. func (h *Header) Logger(logger *zerolog.Logger) *zerolog.Logger { nlogger := logger.With(). diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 2424791f0d..e66d3c9c73 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -741,8 +741,12 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi registry.SetBeaconchain(registry.GetBlockchain()) } + cxPool := core.NewCxPool(core.CxPoolSize) + registry.SetCxPool(cxPool) + // Consensus object. decider := quorum.NewDecider(quorum.SuperMajorityVote, nodeConfig.ShardID) + registry.SetIsBackup(isBackup(hc)) currentConsensus, err := consensus.New( myHost, nodeConfig.ShardID, nodeConfig.ConsensusPriKey, registry, decider, minPeers, aggregateSig) @@ -780,7 +784,7 @@ func setupConsensusAndNode(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfi ) nodeconfig.GetDefaultConfig().DBDir = nodeConfig.DBDir - currentConsensus.SetIsBackup(processNodeType(hc, currentNode)) + processNodeType(hc, currentNode.NodeConfig) currentNode.NodeConfig.SetShardGroupID(nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(nodeConfig.ShardID))) currentNode.NodeConfig.SetClientGroupID(nodeconfig.NewClientGroupIDByShardID(shard.BeaconChainShardID)) currentNode.NodeConfig.ConsensusPriKey = nodeConfig.ConsensusPriKey @@ -829,16 +833,23 @@ func setupTiKV(hc harmonyconfig.HarmonyConfig) shardchain.DBFactory { return factory } -func processNodeType(hc harmonyconfig.HarmonyConfig, currentNode *node.Node) (isBackup bool) { +func processNodeType(hc harmonyconfig.HarmonyConfig, nodeConfig *nodeconfig.ConfigType) { switch hc.General.NodeType { case nodeTypeExplorer: nodeconfig.SetDefaultRole(nodeconfig.ExplorerNode) - currentNode.NodeConfig.SetRole(nodeconfig.ExplorerNode) + nodeConfig.SetRole(nodeconfig.ExplorerNode) case nodeTypeValidator: nodeconfig.SetDefaultRole(nodeconfig.Validator) - currentNode.NodeConfig.SetRole(nodeconfig.Validator) + nodeConfig.SetRole(nodeconfig.Validator) + } +} +func isBackup(hc harmonyconfig.HarmonyConfig) (isBackup bool) { + switch hc.General.NodeType { + case nodeTypeExplorer: + + case nodeTypeValidator: return hc.General.IsBackup } return false diff --git a/consensus/consensus.go b/consensus/consensus.go index 42d57047c0..ddbea9ec46 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -146,16 +146,12 @@ func (consensus *Consensus) ChainReader() engine.ChainReader { return consensus.Blockchain() } -func (consensus *Consensus) GetReadySignal() chan ProposalType { - return consensus.readySignal -} - func (consensus *Consensus) ReadySignal(p ProposalType) { consensus.readySignal <- p } -func (consensus *Consensus) CommitSigChannel() chan []byte { - return consensus.commitSigChannel +func (consensus *Consensus) GetReadySignal() chan ProposalType { + return consensus.readySignal } func (consensus *Consensus) GetCommitSigChannel() chan []byte { @@ -246,14 +242,6 @@ func (consensus *Consensus) getConsensusLeaderPrivateKey() (*bls.PrivateKeyWrapp return consensus.getLeaderPrivateKey(consensus.LeaderPubKey.Object) } -// setBlockVerifier sets the block verifier -func (consensus *Consensus) setBlockVerifier(verifier VerifyBlockFunc) { - consensus.mutex.Lock() - defer consensus.mutex.Unlock() - consensus.BlockVerifier = verifier - consensus.vc.SetVerifyBlock(consensus.verifyBlock) -} - func (consensus *Consensus) IsBackup() bool { return consensus.isBackup } @@ -311,7 +299,9 @@ func New( // Make Sure Verifier is not null consensus.vc = newViewChange() // TODO: reference to blockchain/beaconchain should be removed. - consensus.setBlockVerifier(VerifyNewBlock(registry.GetWebHooks(), consensus.Blockchain(), consensus.Beaconchain())) + verifier := VerifyNewBlock(registry.GetWebHooks(), consensus.Blockchain(), consensus.Beaconchain()) + consensus.BlockVerifier = verifier + consensus.vc.verifyBlock = consensus.verifyBlock // init prometheus metrics initMetrics() @@ -319,3 +309,11 @@ func New( return &consensus, nil } + +func (consensus *Consensus) GetHost() p2p.Host { + return consensus.host +} + +func (consensus *Consensus) Registry() *registry.Registry { + return consensus.registry +} diff --git a/consensus/view_change_construct.go b/consensus/view_change_construct.go index 52ea8bbd72..089ec2ef87 100644 --- a/consensus/view_change_construct.go +++ b/consensus/view_change_construct.go @@ -57,11 +57,6 @@ func newViewChange() *viewChange { return &vc } -// SetVerifyBlock .. -func (vc *viewChange) SetVerifyBlock(verifyBlock VerifyBlockFunc) { - vc.verifyBlock = verifyBlock -} - // AddViewIDKeyIfNotExist .. func (vc *viewChange) AddViewIDKeyIfNotExist(viewID uint64, members multibls.PublicKeys) { if _, ok := vc.bhpSigs[viewID]; !ok { diff --git a/core/blockchain.go b/core/blockchain.go index facb9ed2c6..370f0d09e7 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -214,7 +214,7 @@ type BlockChain interface { // ReadCXReceipts retrieves the cross shard transaction receipts of a given shard. ReadCXReceipts(shardID uint32, blockNum uint64, blockHash common.Hash) (types.CXReceipts, error) // CXMerkleProof calculates the cross shard transaction merkle proof of a given destination shard. - CXMerkleProof(toShardID uint32, block *types.Block) (*types.CXMerkleProof, error) + CXMerkleProof(toShardID uint32, block *block.Header) (*types.CXMerkleProof, error) // WriteCXReceiptsProofSpent mark the CXReceiptsProof list with given unspent status WriteCXReceiptsProofSpent(db rawdb.DatabaseWriter, cxps []*types.CXReceiptsProof) error // IsSpent checks whether a CXReceiptsProof is spent. diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 2c9e306205..8294af26b8 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -2629,10 +2629,10 @@ func (bc *BlockChainImpl) ReadCXReceipts(shardID uint32, blockNum uint64, blockH return cxs, nil } -func (bc *BlockChainImpl) CXMerkleProof(toShardID uint32, block *types.Block) (*types.CXMerkleProof, error) { - proof := &types.CXMerkleProof{BlockNum: block.Number(), BlockHash: block.Hash(), ShardID: block.ShardID(), CXReceiptHash: block.Header().OutgoingReceiptHash(), CXShardHashes: []common.Hash{}, ShardIDs: []uint32{}} +func (bc *BlockChainImpl) CXMerkleProof(toShardID uint32, block *block.Header) (*types.CXMerkleProof, error) { + proof := &types.CXMerkleProof{BlockNum: block.Number(), BlockHash: block.Hash(), ShardID: block.ShardID(), CXReceiptHash: block.OutgoingReceiptHash(), CXShardHashes: []common.Hash{}, ShardIDs: []uint32{}} - epoch := block.Header().Epoch() + epoch := block.Epoch() shardingConfig := shard.Schedule.InstanceForEpoch(epoch) shardNum := int(shardingConfig.NumShards()) diff --git a/core/blockchain_stub.go b/core/blockchain_stub.go index cfe72eed6d..bdab2494e9 100644 --- a/core/blockchain_stub.go +++ b/core/blockchain_stub.go @@ -289,7 +289,7 @@ func (a Stub) ReadCXReceipts(shardID uint32, blockNum uint64, blockHash common.H return nil, errors.Errorf("method ReadCXReceipts not implemented for %s", a.Name) } -func (a Stub) CXMerkleProof(toShardID uint32, block *types.Block) (*types.CXMerkleProof, error) { +func (a Stub) CXMerkleProof(toShardID uint32, block *block.Header) (*types.CXMerkleProof, error) { return nil, errors.Errorf("method CXMerkleProof not implemented for %s", a.Name) } diff --git a/internal/registry/registry.go b/internal/registry/registry.go index 448e148668..7032609103 100644 --- a/internal/registry/registry.go +++ b/internal/registry/registry.go @@ -14,6 +14,8 @@ type Registry struct { beaconchain core.BlockChain webHooks *webhooks.Hooks txPool *core.TxPool + cxPool *core.CxPool + isBackup bool } // New creates a new registry. @@ -88,3 +90,35 @@ func (r *Registry) GetTxPool() *core.TxPool { return r.txPool } + +func (r *Registry) SetIsBackup(isBackup bool) *Registry { + r.mu.Lock() + defer r.mu.Unlock() + + r.isBackup = isBackup + return r +} + +func (r *Registry) IsBackup() bool { + r.mu.Lock() + defer r.mu.Unlock() + + return r.isBackup +} + +// SetCxPool sets the cxpool to registry. +func (r *Registry) SetCxPool(cxPool *core.CxPool) *Registry { + r.mu.Lock() + defer r.mu.Unlock() + + r.cxPool = cxPool + return r +} + +// GetCxPool gets the cxpool from registry. +func (r *Registry) GetCxPool() *core.CxPool { + r.mu.Lock() + defer r.mu.Unlock() + + return r.cxPool +} diff --git a/node/node.go b/node/node.go index 710751776f..06aecb94d9 100644 --- a/node/node.go +++ b/node/node.go @@ -661,6 +661,7 @@ func validateShardBoundMessage(consensus *consensus.Consensus, nodeConfig *nodec } var ( + errMsgHadNoHMYPayLoadAssumption = errors.New("did not have sufficient size for hmy msg") errConsensusMessageOnUnexpectedTopic = errors.New("received consensus on wrong topic") ) @@ -1106,7 +1107,7 @@ func New( node.TxPool = core.NewTxPool(txPoolConfig, node.Blockchain().Config(), blockchain, node.TransactionErrorSink) node.registry.SetTxPool(node.TxPool) - node.CxPool = core.NewCxPool(core.CxPoolSize) + node.CxPool = node.registry.GetCxPool() node.Worker = worker.New(node.Blockchain().Config(), blockchain, beaconChain, engine) node.deciderCache, _ = lru.New(16) diff --git a/node/node_cross_shard.go b/node/node_cross_shard.go index 014432c4ec..e516394834 100644 --- a/node/node_cross_shard.go +++ b/node/node_cross_shard.go @@ -3,6 +3,8 @@ package node import ( "github.com/ethereum/go-ethereum/rlp" proto_node "github.com/harmony-one/harmony/api/proto/node" + "github.com/harmony-one/harmony/block" + "github.com/harmony-one/harmony/consensus" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" @@ -13,7 +15,7 @@ import ( // BroadcastCXReceipts broadcasts cross shard receipts to correspoding // destination shards -func (node *Node) BroadcastCXReceipts(newBlock *types.Block) { +func BroadcastCXReceipts(newBlock *types.Block, consensus *consensus.Consensus) { commitSigAndBitmap := newBlock.GetCurrentCommitSig() //#### Read payload data from committed msg if len(commitSigAndBitmap) <= 96 { @@ -31,27 +33,27 @@ func (node *Node) BroadcastCXReceipts(newBlock *types.Block) { epoch := newBlock.Header().Epoch() shardingConfig := shard.Schedule.InstanceForEpoch(epoch) shardNum := int(shardingConfig.NumShards()) - myShardID := node.Consensus.ShardID + myShardID := consensus.ShardID utils.Logger().Info().Int("shardNum", shardNum).Uint32("myShardID", myShardID).Uint64("blockNum", newBlock.NumberU64()).Msg("[BroadcastCXReceipts]") for i := 0; i < shardNum; i++ { if i == int(myShardID) { continue } - node.BroadcastCXReceiptsWithShardID(newBlock, commitSig, commitBitmap, uint32(i)) + BroadcastCXReceiptsWithShardID(newBlock.Header(), commitSig, commitBitmap, uint32(i), consensus) } } // BroadcastCXReceiptsWithShardID broadcasts cross shard receipts to given ToShardID -func (node *Node) BroadcastCXReceiptsWithShardID(block *types.Block, commitSig []byte, commitBitmap []byte, toShardID uint32) { - myShardID := node.Consensus.ShardID +func BroadcastCXReceiptsWithShardID(block *block.Header, commitSig []byte, commitBitmap []byte, toShardID uint32, consensus *consensus.Consensus) { + myShardID := consensus.ShardID utils.Logger().Debug(). Uint32("toShardID", toShardID). Uint32("myShardID", myShardID). Uint64("blockNum", block.NumberU64()). Msg("[BroadcastCXReceiptsWithShardID]") - cxReceipts, err := node.Blockchain().ReadCXReceipts(toShardID, block.NumberU64(), block.Hash()) + cxReceipts, err := consensus.Blockchain().ReadCXReceipts(toShardID, block.NumberU64(), block.Hash()) if err != nil || len(cxReceipts) == 0 { utils.Logger().Debug().Uint32("ToShardID", toShardID). Int("numCXReceipts", len(cxReceipts)). @@ -59,7 +61,7 @@ func (node *Node) BroadcastCXReceiptsWithShardID(block *types.Block, commitSig [ return } - merkleProof, err := node.Blockchain().CXMerkleProof(toShardID, block) + merkleProof, err := consensus.Blockchain().CXMerkleProof(toShardID, block) if err != nil { utils.Logger().Warn(). Uint32("ToShardID", toShardID). @@ -70,7 +72,7 @@ func (node *Node) BroadcastCXReceiptsWithShardID(block *types.Block, commitSig [ cxReceiptsProof := &types.CXReceiptsProof{ Receipts: cxReceipts, MerkleProof: merkleProof, - Header: block.Header(), + Header: block, CommitSig: commitSig, CommitBitmap: commitBitmap, } @@ -81,36 +83,40 @@ func (node *Node) BroadcastCXReceiptsWithShardID(block *types.Block, commitSig [ Interface("cxp", cxReceiptsProof). Msg("[BroadcastCXReceiptsWithShardID] ReadCXReceipts and MerkleProof ready. Sending CX receipts...") // TODO ek – limit concurrency - go node.host.SendMessageToGroups([]nodeconfig.GroupID{groupID}, + go consensus.GetHost().SendMessageToGroups([]nodeconfig.GroupID{groupID}, p2p.ConstructMessage(proto_node.ConstructCXReceiptsProof(cxReceiptsProof)), ) } // BroadcastMissingCXReceipts broadcasts missing cross shard receipts per request -func (node *Node) BroadcastMissingCXReceipts() { - sendNextTime := []core.CxEntry{} - it := node.CxPool.Pool().Iterator() +func BroadcastMissingCXReceipts(c *consensus.Consensus) { + var ( + sendNextTime = make([]core.CxEntry, 0) + cxPool = c.Registry().GetCxPool() + blockchain = c.Blockchain() + ) + it := cxPool.Pool().Iterator() for entry := range it.C { cxEntry := entry.(core.CxEntry) toShardID := cxEntry.ToShardID - blk := node.Blockchain().GetBlockByHash(cxEntry.BlockHash) + blk := blockchain.GetBlockByHash(cxEntry.BlockHash) if blk == nil { continue } blockNum := blk.NumberU64() - nextHeader := node.Blockchain().GetHeaderByNumber(blockNum + 1) + nextHeader := blockchain.GetHeaderByNumber(blockNum + 1) if nextHeader == nil { sendNextTime = append(sendNextTime, cxEntry) continue } sig := nextHeader.LastCommitSignature() bitmap := nextHeader.LastCommitBitmap() - node.BroadcastCXReceiptsWithShardID(blk, sig[:], bitmap, toShardID) + BroadcastCXReceiptsWithShardID(blk.Header(), sig[:], bitmap, toShardID, c) } - node.CxPool.Clear() + cxPool.Clear() // this should not happen or maybe happen for impatient user for _, entry := range sendNextTime { - node.CxPool.Add(entry) + cxPool.Add(entry) } } diff --git a/node/node_handler.go b/node/node_handler.go index a7e496e98a..92c3396d4b 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -139,8 +139,8 @@ func (node *Node) stakingMessageHandler(msgPayload []byte) { // BroadcastNewBlock is called by consensus leader to sync new blocks with other clients/nodes. // NOTE: For now, just send to the client (basically not broadcasting) // TODO (lc): broadcast the new blocks to new nodes doing state sync -func (node *Node) BroadcastNewBlock(newBlock *types.Block) { - groups := []nodeconfig.GroupID{node.NodeConfig.GetClientGroupID()} +func (node *Node) BroadcastNewBlock(newBlock *types.Block, nodeConfig *nodeconfig.ConfigType) { + groups := []nodeconfig.GroupID{nodeConfig.GetClientGroupID()} utils.Logger().Info(). Msgf( "broadcasting new block %d, group %s", newBlock.NumberU64(), groups[0], @@ -335,9 +335,9 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block) error { if node.IsRunningBeaconChain() { // TODO: consider removing this and letting other nodes broadcast new blocks. // But need to make sure there is at least 1 node that will do the job. - node.BroadcastNewBlock(newBlock) + node.BroadcastNewBlock(newBlock, node.NodeConfig) } - node.BroadcastCXReceipts(newBlock) + BroadcastCXReceipts(newBlock, node.Consensus) } else { if node.Consensus.Mode() != consensus.Listening { numSignatures := node.Consensus.NumSignaturesIncludedInBlock(newBlock) @@ -358,15 +358,15 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block) error { if rnd < 1 { // Beacon validators also broadcast new blocks to make sure beacon sync is strong. if node.IsRunningBeaconChain() { - node.BroadcastNewBlock(newBlock) + node.BroadcastNewBlock(newBlock, node.NodeConfig) } - node.BroadcastCXReceipts(newBlock) + BroadcastCXReceipts(newBlock, node.Consensus) } } } // Broadcast client requested missing cross shard receipts if there is any - node.BroadcastMissingCXReceipts() + BroadcastMissingCXReceipts(node.Consensus) if h := node.NodeConfig.WebHooks.Hooks; h != nil { if h.Availability != nil { diff --git a/node/node_newblock.go b/node/node_newblock.go index 27c0d843bd..008182fbcc 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -76,7 +76,7 @@ func (node *Node) WaitForConsensusReadyV2(cs *consensus.Consensus, stopChan chan } else { newCommitSigsChan <- sigs } - case commitSigs := <-cs.CommitSigChannel(): + case commitSigs := <-cs.GetCommitSigChannel(): utils.Logger().Info().Msg("[ProposeNewBlock] received commit sigs asynchronously") if len(commitSigs) > bls.BLSSignatureSizeInBytes { newCommitSigsChan <- commitSigs From 8e076e60a271a44bd8f1e088b4d472bce0ea7822 Mon Sep 17 00:00:00 2001 From: Casey Gardiner <117784577+ONECasey@users.noreply.github.com> Date: Thu, 8 Jun 2023 08:17:43 -0700 Subject: [PATCH 415/420] Rebase dev to current main (#4443) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Release Candidate 2023.2.0 ( dev -> main ) (#4399) * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * activate epoch * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review "--port flag". * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * Commented golangci. * staged stream sync v1.0 * fix protocol tests * fix spell * remove unused struct * fix rosetta test * add comments and refactor verify sig * add comments, remove extra function * add comment * refactor errors, rename metrics * refactor p2p host creation * fix initsync and host creation * fix short range hash chain * fix beacon node detection for p2p protocol * refactor stream peer cooldown and fix protocol beacon node field * refactor p2p host and routing * fix p2p discovery test issue * add MaxAdvertiseWaitTime to handle advertisements interval and address stream connection issue * terminal print the peer id and proto id * fix boot complete message when node is shut down * add new config option ( ForceReachabilityPublic ) to fix local-net consensus issue * fix self query issue * fix test NewDNSSyncingPeerProvider * [testnet] disable leader rotation * fix discovery issue for legacy sync * add watermark low/high options for p2p connection manager * add test for new conn manager flags * fix dedent * add comment to inform about p2p connection manager options * fix max height issue * add a separate log for get max height error * fix log * feat: triesInMemory flag * fix: panic if TriesInMemory is 1 to 2 * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed timers locks. * Removed fbft locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional logs. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * Fix formatting. * Rebased onto dev. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Fix review comment. * Go mod tidy. * Set to EpochTBD. * Fix tests. * [core] fix state handling of self destruct If a contract self destructs to self and then receives funds within the same transaction, it is possible for its stale state to be saved. This change removes that possibility by checking for deleted state objects before returning them. * Fixed race error. * rpc: add configurable http and `eth_call` timeout * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fixed code review. * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * staged stream sync v1.0 * refactor errors, rename metrics * fix p2p discovery test issue * add watermark low/high options for p2p connection manager * fix dedent * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Go mod tidy. * remove default timeouts * Rotate external leaders on non-beacon chains. * Fix nil panic. * Fixes. * Update singleton.go * evm: don't return extcode for validators Due to technical debt, validator information is stored in the code field of the address. The code field can be accessed in Solidity for an arbitrary address using `extcodesize`, `extcodehash`, and `extcodecopy` or helper commands (such as `address.code.Length`). The presence of this field is used by contract developers to (erroneously) deny smart contract access to other smart contracts (and therefore, validators). This PR fixes that oversight by returning the same values as other EOAs for known validator addresses. Obviously, it needs a hard fork that will be scheduled separately. * Fix context passing. * Clean up code. * Removed engine dependency. * Fix possible panic. * Clean up code. * Network type. * Fix tests. * Revert "Removed engine dependency." (#4392) * Revert "Fix tests." This reverts commit 597ba2d6f1ed54ff599b9d9b8940c7285ab1277a. * Revert "Network type." This reverts commit 5e1878aedca3989dc0f34161dae1833e43ca6a76. * Revert "Clean up code." This reverts commit 15885f4c9b9263746827172b4f4f56d0926d18e2. * Revert "Fix possible panic." This reverts commit 1a70d5eb66cdbf8a23791806b71a323eed320085. * Revert "Removed engine dependency." This reverts commit 8c2ff803f709f944cfc8b1278f35cf5b2cacf859. * gitignore the cache folder (#4389) * stable localnet with external validator (#4388) * stable localnet with external validator * ignore deploy config file comments * reduce node launched in localnet * update makefile * localnet configuration - add more fn * fix validator information command typo * Configurable tx pool. (#4240) * AccountQueue & GlobalQueue. * Lifetime duration. * [pool] make flags configurable * [pool] use 4096 as default `GlobalSlots` * [rosetta] update default values of tx pool * [test] update value to default * PriceLimit and PriceBump. * Fix tests. * Fix price limit & bump. * Updated, fixed migrate version and tests. * Rebased. * Fix go toml version. --------- Co-authored-by: Konstantin Co-authored-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> * Upgrade rawdb and statedb codes to add the latest functionalities of ethdb (#4374) * added bloom filter * upgrade rawdb and statedb * change var name and remove extra comments * return back fake storage in case if we need it for test later * add the previous change back * remove some extra entries from go.mod * fix WritePreimages to use batch * mark unused functions which are ported over from eth --------- Co-authored-by: Casey Gardiner <117784577+ONECasey@users.noreply.github.com> * update all ethereum rawdb pointers to use harmony rawdb (#4395) * Fix reduce node dependencies. (#4379) * Fix. * Fix. * Fix pool init. * Clean up. * add prefix for contract code (#4397) * Rotate external validators for non-beacon shards. (#4373) * Rotate only non beacon shards. * Rotate all shards, but only hmy validators for beacon. * Fix type. * Revert "Fix type." This reverts commit 0a8b506c763d9f8609abff7395ba32b18e43b149. * Revert "Rotate all shards, but only hmy validators for beacon." This reverts commit 70b09e2de81aa2cbffae3ccdfd4e334e7d938759. * Fixed failed test. * Revert "Revert "Rotate all shards, but only hmy validators for beacon."" This reverts commit 66cfaa9817488be60ed5b5cfee1fe833ede237c8. * Frequency by slots count. * Fix config. * First validator produce rest blocks. * Updated. * Add lock. * Add prefix for validator wrapper (#4402) * add separate prefix for validator wrapper * update comments * make read/write backward compatible * add validator codes to stats * goimports * goimports accessor_state * add snapshot feature to state db (#4406) * Typed cache & Node cleanup. (#4409) * Channels usage through methods. * Fix retry count. Removed proposedBlock. * keysToAddrs rewritten to lrucache. * core, internal/configs: HIP28-v2 fee collection (#4410) * core, internal/configs: HIP28-v2 fee collection Based on the Snapshot vote that has passed, collect 50% of the fee to a community maintained account and the remainder to an account used to pay for infrastructure costs. Note that these accounts need to be decided and set in the code at this moment, and the feature needs to be activated by setting the `FeeCollectEpoch` of the `ChainConfig` object. The implementation for devnet is a bit different than compared to others because the feature was activated on devnet with 100% collection to an account. I have handled this case separately in `devnet.go`. * test: add test for StateTransition.ApplyMessage The objective of this unit test is to check that the fees of a transaction are appropriately credited to the fee collectors that are set. This means, for a transaction of 21,000 gas limit and 100 gwei gas price, two equal fee collectors get 10,500 * 100 gwei each. In addition, to be clear that the refund mechanism (in case a transaction with extra gas comes in) works, the tested transaction has a 50,000 gas limit of which only 21,000 gas limit is actually consumed. * sharding/config: clarify local fee collector pk * sharding/config: set testnet fee collector same as devnet * test: add test for truncated fee distribution * sharding/config: set fee collector addresses * test: hardcode the expected fee collected * goimports * params/config: set testnet fee collect epoch Schedule testnet hard fork epoch to be 1296, which begins around the time 2023-04-28 07:14:20+00:00 * params/config: schedule devnee fee collection Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> * Minor: removed time.Sleep from tests. (#4412) * Provide current time as argument. * Fix test. * Fix naming. * Mainnet Release Candidate 2023.1.2 (#4376) (#4414) * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * Bump github.com/aws/aws-sdk-go from 1.33.0 to 1.34.0 Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.33.0 to 1.34.0. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/v1.34.0/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.33.0...v1.34.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production ... * Bump github.com/ipld/go-ipld-prime from 0.9.0 to 0.19.0 Bumps [github.com/ipld/go-ipld-prime](https://github.com/ipld/go-ipld-prime) from 0.9.0 to 0.19.0. - [Release notes](https://github.com/ipld/go-ipld-prime/releases) - [Changelog](https://github.com/ipld/go-ipld-prime/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipld/go-ipld-prime/compare/v0.9.0...v0.19.0) --- updated-dependencies: - dependency-name: github.com/ipld/go-ipld-prime dependency-type: indirect ... * Bump golang.org/x/net from 0.3.0 to 0.7.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.3.0 to 0.7.0. - [Release notes](https://github.com/golang/net/releases) - [Commits](https://github.com/golang/net/compare/v0.3.0...v0.7.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... * Small fixes. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * activate epoch * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review "--port flag". * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * Commented golangci. * staged stream sync v1.0 * fix protocol tests * fix spell * remove unused struct * fix rosetta test * add comments and refactor verify sig * add comments, remove extra function * add comment * refactor errors, rename metrics * refactor p2p host creation * fix initsync and host creation * fix short range hash chain * fix beacon node detection for p2p protocol * refactor stream peer cooldown and fix protocol beacon node field * refactor p2p host and routing * fix p2p discovery test issue * add MaxAdvertiseWaitTime to handle advertisements interval and address stream connection issue * terminal print the peer id and proto id * fix boot complete message when node is shut down * add new config option ( ForceReachabilityPublic ) to fix local-net consensus issue * fix self query issue * fix test NewDNSSyncingPeerProvider * [testnet] disable leader rotation * fix discovery issue for legacy sync * add watermark low/high options for p2p connection manager * add test for new conn manager flags * fix dedent * add comment to inform about p2p connection manager options * fix max height issue * add a separate log for get max height error * fix log * feat: triesInMemory flag * fix: panic if TriesInMemory is 1 to 2 * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed timers locks. * Removed fbft locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional logs. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * Fix formatting. * Rebased onto dev. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Fix review comment. * Go mod tidy. * Set to EpochTBD. * Fix tests. * [core] fix state handling of self destruct If a contract self destructs to self and then receives funds within the same transaction, it is possible for its stale state to be saved. This change removes that possibility by checking for deleted state objects before returning them. * Fixed race error. * rpc: add configurable http and `eth_call` timeout * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fixed code review. * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * staged stream sync v1.0 * refactor errors, rename metrics * fix p2p discovery test issue * add watermark low/high options for p2p connection manager * fix dedent * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Go mod tidy. * remove default timeouts * Rotate external leaders on non-beacon chains. * Fix nil panic. * Fixes. * Update singleton.go * evm: don't return extcode for validators Due to technical debt, validator information is stored in the code field of the address. The code field can be accessed in Solidity for an arbitrary address using `extcodesize`, `extcodehash`, and `extcodecopy` or helper commands (such as `address.code.Length`). The presence of this field is used by contract developers to (erroneously) deny smart contract access to other smart contracts (and therefore, validators). This PR fixes that oversight by returning the same values as other EOAs for known validator addresses. Obviously, it needs a hard fork that will be scheduled separately. * Fix context passing. * Clean up code. * Removed engine dependency. * Fix possible panic. * Clean up code. * Network type. * Fix tests. * Revert "Removed engine dependency." (#4392) * Revert "Fix tests." This reverts commit 597ba2d6f1ed54ff599b9d9b8940c7285ab1277a. * Revert "Network type." This reverts commit 5e1878aedca3989dc0f34161dae1833e43ca6a76. * Revert "Clean up code." This reverts commit 15885f4c9b9263746827172b4f4f56d0926d18e2. * Revert "Fix possible panic." This reverts commit 1a70d5eb66cdbf8a23791806b71a323eed320085. * Revert "Removed engine dependency." This reverts commit 8c2ff803f709f944cfc8b1278f35cf5b2cacf859. * gitignore the cache folder (#4389) * stable localnet with external validator (#4388) * stable localnet with external validator * ignore deploy config file comments * reduce node launched in localnet * update makefile * localnet configuration - add more fn * fix validator information command typo --------- Signed-off-by: dependabot[bot] Co-authored-by: Casey Gardiner <117784577+ONECasey@users.noreply.github.com> Co-authored-by: frozen <355847+Frozen@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Soph <35721420+sophoah@users.noreply.github.com> * chore: merge `main` into `dev` (#4415) * Mainnet Release Candidate 2023.1.2 (#4376) * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * Bump github.com/aws/aws-sdk-go from 1.33.0 to 1.34.0 Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.33.0 to 1.34.0. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/v1.34.0/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.33.0...v1.34.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Bump github.com/ipld/go-ipld-prime from 0.9.0 to 0.19.0 Bumps [github.com/ipld/go-ipld-prime](https://github.com/ipld/go-ipld-prime) from 0.9.0 to 0.19.0. - [Release notes](https://github.com/ipld/go-ipld-prime/releases) - [Changelog](https://github.com/ipld/go-ipld-prime/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipld/go-ipld-prime/compare/v0.9.0...v0.19.0) --- updated-dependencies: - dependency-name: github.com/ipld/go-ipld-prime dependency-type: indirect ... Signed-off-by: dependabot[bot] * Bump golang.org/x/net from 0.3.0 to 0.7.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.3.0 to 0.7.0. - [Release notes](https://github.com/golang/net/releases) - [Commits](https://github.com/golang/net/compare/v0.3.0...v0.7.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] * Small fixes. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * activate epoch * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review "--port flag". * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * Commented golangci. * staged stream sync v1.0 * fix protocol tests * fix spell * remove unused struct * fix rosetta test * add comments and refactor verify sig * add comments, remove extra function * add comment * refactor errors, rename metrics * refactor p2p host creation * fix initsync and host creation * fix short range hash chain * fix beacon node detection for p2p protocol * refactor stream peer cooldown and fix protocol beacon node field * refactor p2p host and routing * fix p2p discovery test issue * add MaxAdvertiseWaitTime to handle advertisements interval and address stream connection issue * terminal print the peer id and proto id * fix boot complete message when node is shut down * add new config option ( ForceReachabilityPublic ) to fix local-net consensus issue * fix self query issue * fix test NewDNSSyncingPeerProvider * [testnet] disable leader rotation * fix discovery issue for legacy sync * add watermark low/high options for p2p connection manager * add test for new conn manager flags * fix dedent * add comment to inform about p2p connection manager options * fix max height issue * add a separate log for get max height error * fix log * feat: triesInMemory flag * fix: panic if TriesInMemory is 1 to 2 * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed timers locks. * Removed fbft locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional logs. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * Fix formatting. * Rebased onto dev. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Fix review comment. * Go mod tidy. * Set to EpochTBD. * Fix tests. * [core] fix state handling of self destruct If a contract self destructs to self and then receives funds within the same transaction, it is possible for its stale state to be saved. This change removes that possibility by checking for deleted state objects before returning them. * Fixed race error. * rpc: add configurable http and `eth_call` timeout * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fixed code review. * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * staged stream sync v1.0 * refactor errors, rename metrics * fix p2p discovery test issue * add watermark low/high options for p2p connection manager * fix dedent * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Go mod tidy. * remove default timeouts * Rotate external leaders on non-beacon chains. * Fix nil panic. * Fixes. * Update singleton.go * evm: don't return extcode for validators Due to technical debt, validator information is stored in the code field of the address. The code field can be accessed in Solidity for an arbitrary address using `extcodesize`, `extcodehash`, and `extcodecopy` or helper commands (such as `address.code.Length`). The presence of this field is used by contract developers to (erroneously) deny smart contract access to other smart contracts (and therefore, validators). This PR fixes that oversight by returning the same values as other EOAs for known validator addresses. Obviously, it needs a hard fork that will be scheduled separately. * Fix context passing. * Clean up code. * Removed engine dependency. * Fix possible panic. * Clean up code. * Network type. * Fix tests. * Revert "Removed engine dependency." (#4392) * Revert "Fix tests." This reverts commit 597ba2d6f1ed54ff599b9d9b8940c7285ab1277a. * Revert "Network type." This reverts commit 5e1878aedca3989dc0f34161dae1833e43ca6a76. * Revert "Clean up code." This reverts commit 15885f4c9b9263746827172b4f4f56d0926d18e2. * Revert "Fix possible panic." This reverts commit 1a70d5eb66cdbf8a23791806b71a323eed320085. * Revert "Removed engine dependency." This reverts commit 8c2ff803f709f944cfc8b1278f35cf5b2cacf859. * gitignore the cache folder (#4389) * stable localnet with external validator (#4388) * stable localnet with external validator * ignore deploy config file comments * reduce node launched in localnet * update makefile * localnet configuration - add more fn * fix validator information command typo --------- Signed-off-by: dependabot[bot] Co-authored-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Co-authored-by: frozen <355847+Frozen@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Soph <35721420+sophoah@users.noreply.github.com> * build: update pinned curl version (#4394) Per the Alpine Linux package repositories, the version for cURL included with v3.16 has changed to revision 6 * consensus: replace type assert with test (#4398) If `consensus.finalityCounter` does not have anything stored (for example in Syncing mode), the `Load()` returns an interface that cannot be automatically asserted to an `int64`. This results in the node crashing. This commit fixes that. * Turn pprof default on with local saved files (#3894) * Turn pprof default on with local saved files * [pprof] change interval from 600s to 3600s * Revert "Turn pprof default on with local saved files (#3894)" (#4400) This reverts commit 78d26d7910a58ded3bfe689b3de07ea28d95d7d5. --------- Signed-off-by: dependabot[bot] Co-authored-by: Casey Gardiner <117784577+ONECasey@users.noreply.github.com> Co-authored-by: frozen <355847+Frozen@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Soph <35721420+sophoah@users.noreply.github.com> Co-authored-by: Jacky Wang * internal/params: set validator code fix hard forks (#4413) * internal/params: schedule hard forks Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> * internal/params: set localnet fee collect epoch 2 Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> --------- Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> * internal/params: schedule HIP28v2 + val code fix (#4416) Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> * Dev fix conflicts. (#4417) * Mainnet Release Candidate 2023.1.2 (#4376) * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * Bump github.com/aws/aws-sdk-go from 1.33.0 to 1.34.0 Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.33.0 to 1.34.0. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/v1.34.0/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.33.0...v1.34.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Bump github.com/ipld/go-ipld-prime from 0.9.0 to 0.19.0 Bumps [github.com/ipld/go-ipld-prime](https://github.com/ipld/go-ipld-prime) from 0.9.0 to 0.19.0. - [Release notes](https://github.com/ipld/go-ipld-prime/releases) - [Changelog](https://github.com/ipld/go-ipld-prime/blob/master/CHANGELOG.md) - [Commits](https://github.com/ipld/go-ipld-prime/compare/v0.9.0...v0.19.0) --- updated-dependencies: - dependency-name: github.com/ipld/go-ipld-prime dependency-type: indirect ... Signed-off-by: dependabot[bot] * Bump golang.org/x/net from 0.3.0 to 0.7.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.3.0 to 0.7.0. - [Release notes](https://github.com/golang/net/releases) - [Commits](https://github.com/golang/net/compare/v0.3.0...v0.7.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] * Small fixes. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * activate epoch * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review "--port flag". * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * Commented golangci. * staged stream sync v1.0 * fix protocol tests * fix spell * remove unused struct * fix rosetta test * add comments and refactor verify sig * add comments, remove extra function * add comment * refactor errors, rename metrics * refactor p2p host creation * fix initsync and host creation * fix short range hash chain * fix beacon node detection for p2p protocol * refactor stream peer cooldown and fix protocol beacon node field * refactor p2p host and routing * fix p2p discovery test issue * add MaxAdvertiseWaitTime to handle advertisements interval and address stream connection issue * terminal print the peer id and proto id * fix boot complete message when node is shut down * add new config option ( ForceReachabilityPublic ) to fix local-net consensus issue * fix self query issue * fix test NewDNSSyncingPeerProvider * [testnet] disable leader rotation * fix discovery issue for legacy sync * add watermark low/high options for p2p connection manager * add test for new conn manager flags * fix dedent * add comment to inform about p2p connection manager options * fix max height issue * add a separate log for get max height error * fix log * feat: triesInMemory flag * fix: panic if TriesInMemory is 1 to 2 * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed timers locks. * Removed fbft locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional logs. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * Fix formatting. * Rebased onto dev. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * comment activation * 295 epoch * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Fix review comment. * Go mod tidy. * Set to EpochTBD. * Fix tests. * [core] fix state handling of self destruct If a contract self destructs to self and then receives funds within the same transaction, it is possible for its stale state to be saved. This change removes that possibility by checking for deleted state objects before returning them. * Fixed race error. * rpc: add configurable http and `eth_call` timeout * remove default timeouts * store the evm call timeout in rosetta object * [cmd] actually apply ToRPCServerConfig * Removed unused method. * Rotate external leaders on non-beacon chains. * Fix nil panic. * in progress. * in progress. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fixed code review. * Fix review comments. * Returned locks in rotateLeader. * Rebased onto dev. * staged stream sync v1.0 * refactor errors, rename metrics * fix p2p discovery test issue * add watermark low/high options for p2p connection manager * fix dedent * in progress. * consensus check is forked * fix * Cleanup and fix update pub keys. * fix fix fix fix fix * activate epoch * EpochTBD for leader rotation epoch. * 295 epoch * Decider no longer requires public keys as a dependency. (#4289) * Consensus doesn't require anymore `Node` as a circular dependency. * Proper blockchain initialization. * Rwlock consensus. * Removed channels. * Removed view change locks. * Removed multiSigMutex locks. * Removed leader locks. * Removed additional locks and isViewChange. * Added locks detected by race. * Added locks detected by race. * Locks for start. * Removed additional locks. * Removed additional locks. * Make func private. * Make VerifyBlock private. * Make IsLeader private. * Make ParseFBFTMessage private. * Fix remove locks. * Added additional locks. * Added additional locks. * Added readSignatureBitmapPayload locks. * Added HandleMessageUpdate locks. * Added LastMile locks. * Locks for IsValidatorInCommittee. * Fixed locks. * Fixed tests. * Fixed lock. * Rebased over leader rotation. * in progress. * consensus check is forked * update master * fix leader * check leader for N blocks * fix * fix * Cleanup and fix update pub keys. * Rotate leader. * fix fix fix fix fix * Cleaned. * Cache for `GetLeaderPubKeyFromCoinbase`, removed `NthNextHmyExt`. * Fix failed tests. * Fixed code review. * Fix review comments. * Merged leader rotation. * Rebased on dev. * Rebased on dev. * Fix usage of private methods. * Fix usage of private methods. * Fix usage of private methods. * Removed deadcode, LockedFBFTPhase. * Fix review comment. * Go mod tidy. * remove default timeouts * Rotate external leaders on non-beacon chains. * Fix nil panic. * Fixes. * Update singleton.go * evm: don't return extcode for validators Due to technical debt, validator information is stored in the code field of the address. The code field can be accessed in Solidity for an arbitrary address using `extcodesize`, `extcodehash`, and `extcodecopy` or helper commands (such as `address.code.Length`). The presence of this field is used by contract developers to (erroneously) deny smart contract access to other smart contracts (and therefore, validators). This PR fixes that oversight by returning the same values as other EOAs for known validator addresses. Obviously, it needs a hard fork that will be scheduled separately. * Fix context passing. * Clean up code. * Removed engine dependency. * Fix possible panic. * Clean up code. * Network type. * Fix tests. * Revert "Removed engine dependency." (#4392) * Revert "Fix tests." This reverts commit 597ba2d6f1ed54ff599b9d9b8940c7285ab1277a. * Revert "Network type." This reverts commit 5e1878aedca3989dc0f34161dae1833e43ca6a76. * Revert "Clean up code." This reverts commit 15885f4c9b9263746827172b4f4f56d0926d18e2. * Revert "Fix possible panic." This reverts commit 1a70d5eb66cdbf8a23791806b71a323eed320085. * Revert "Removed engine dependency." This reverts commit 8c2ff803f709f944cfc8b1278f35cf5b2cacf859. * gitignore the cache folder (#4389) * stable localnet with external validator (#4388) * stable localnet with external validator * ignore deploy config file comments * reduce node launched in localnet * update makefile * localnet configuration - add more fn * fix validator information command typo --------- Signed-off-by: dependabot[bot] Co-authored-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Co-authored-by: frozen <355847+Frozen@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Soph <35721420+sophoah@users.noreply.github.com> * build: update pinned curl version (#4394) Per the Alpine Linux package repositories, the version for cURL included with v3.16 has changed to revision 6 * consensus: replace type assert with test (#4398) If `consensus.finalityCounter` does not have anything stored (for example in Syncing mode), the `Load()` returns an interface that cannot be automatically asserted to an `int64`. This results in the node crashing. This commit fixes that. * Turn pprof default on with local saved files (#3894) * Turn pprof default on with local saved files * [pprof] change interval from 600s to 3600s * Revert "Turn pprof default on with local saved files (#3894)" (#4400) This reverts commit 78d26d7910a58ded3bfe689b3de07ea28d95d7d5. * go mod tidy. * Increased wait time. --------- Signed-off-by: dependabot[bot] Co-authored-by: Casey Gardiner <117784577+ONECasey@users.noreply.github.com> Co-authored-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Soph <35721420+sophoah@users.noreply.github.com> Co-authored-by: Jacky Wang --------- Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Signed-off-by: dependabot[bot] Co-authored-by: frozen <355847+Frozen@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Soph <35721420+sophoah@users.noreply.github.com> Co-authored-by: Konstantin Co-authored-by: Gheis Mohammadi Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jacky Wang * internal/params: un-schedule mainnet hard forks (#4420) * internal/params: un-schedule mainnet hard forks and add logs to help debug the recent consensus loss * core/state: update logs * [hotfix] fix code hash conflicts (#4431) * fix code hash issue * goimports --------- Signed-off-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Signed-off-by: dependabot[bot] Co-authored-by: frozen <355847+Frozen@users.noreply.github.com> Co-authored-by: “GheisMohammadi” <“Gheis.Mohammadi@gmail.com”> Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> Co-authored-by: MaxMustermann2 <82761650+MaxMustermann2@users.noreply.github.com> Co-authored-by: Sun Hyuk Ahn Co-authored-by: Soph <35721420+sophoah@users.noreply.github.com> Co-authored-by: Konstantin Co-authored-by: Gheis Mohammadi Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jacky Wang --- core/state/snapshot/snapshot.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 210fcac62b..d294ff1857 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -206,6 +206,7 @@ func New(config Config, diskdb ethdb.KeyValueStore, triedb *trie.Database, root utils.Logger().Warn().Err(err).Msg("Snapshot maintenance disabled (syncing)") return snap, nil } + // Create the building waiter if the background generation is allowed if !config.NoBuild && !config.AsyncBuild { defer snap.waitBuild() From 098900291c6bbf06036147ff0a93cb5f61a1a024 Mon Sep 17 00:00:00 2001 From: Konstantin <355847+Frozen@users.noreply.github.com> Date: Fri, 9 Jun 2023 11:51:30 -0400 Subject: [PATCH 416/420] Fix compile error. (#4444) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix compile error. * fix memdb temp path --------- Co-authored-by: “GheisMohammadi” <36589218+GheisMohammadi@users.noreply.github.com> --- api/service/stagedstreamsync/syncing.go | 20 +++++--- go.mod | 52 +++++++++++---------- go.sum | 62 +++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 30 deletions(-) diff --git a/api/service/stagedstreamsync/syncing.go b/api/service/stagedstreamsync/syncing.go index 98b74cf0d0..75a15ff55a 100644 --- a/api/service/stagedstreamsync/syncing.go +++ b/api/service/stagedstreamsync/syncing.go @@ -52,14 +52,14 @@ func CreateStagedSync(ctx context.Context, var mainDB kv.RwDB dbs := make([]kv.RwDB, config.Concurrency) if UseMemDB { - mainDB = memdb.New() + mainDB = memdb.New(getMemDbTempPath(dbDir, -1)) for i := 0; i < config.Concurrency; i++ { - dbs[i] = memdb.New() + dbs[i] = memdb.New(getMemDbTempPath(dbDir, i)) } } else { - mainDB = mdbx.NewMDBX(log.New()).Path(GetBlockDbPath(isBeacon, -1, dbDir)).MustOpen() + mainDB = mdbx.NewMDBX(log.New()).Path(getBlockDbPath(isBeacon, -1, dbDir)).MustOpen() for i := 0; i < config.Concurrency; i++ { - dbPath := GetBlockDbPath(isBeacon, i, dbDir) + dbPath := getBlockDbPath(isBeacon, i, dbDir) dbs[i] = mdbx.NewMDBX(log.New()).Path(dbPath).MustOpen() } } @@ -138,8 +138,16 @@ func initDB(ctx context.Context, mainDB kv.RwDB, dbs []kv.RwDB, concurrency int) return nil } -// GetBlockDbPath returns the path of the cache database which stores blocks -func GetBlockDbPath(beacon bool, loopID int, dbDir string) string { +// getMemDbTempPath returns the path of the temporary cache database for memdb +func getMemDbTempPath(dbDir string, dbIndex int) string { + if dbIndex >= 0 { + return fmt.Sprintf("%s_%d", filepath.Join(dbDir, "cache/memdb/db"), dbIndex) + } + return filepath.Join(dbDir, "cache/memdb/db_main") +} + +// getBlockDbPath returns the path of the cache database which stores blocks +func getBlockDbPath(beacon bool, loopID int, dbDir string) string { if beacon { if loopID >= 0 { return fmt.Sprintf("%s_%d", filepath.Join(dbDir, "cache/beacon_blocks_db"), loopID) diff --git a/go.mod b/go.mod index 4dd98958b3..32ac5feea0 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/ethereum/go-ethereum v1.11.2 github.com/go-redis/redis/v8 v8.11.5 github.com/golang/mock v1.6.0 - github.com/golang/protobuf v1.5.2 + github.com/golang/protobuf v1.5.3 github.com/golangci/golangci-lint v1.22.2 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 @@ -46,20 +46,20 @@ require ( github.com/spf13/cobra v1.5.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.14.0 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.4 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/tikv/client-go/v2 v2.0.1 github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee go.uber.org/ratelimit v0.1.0 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.6.0 - golang.org/x/net v0.7.0 // indirect - golang.org/x/sync v0.1.0 - golang.org/x/sys v0.5.0 // indirect - golang.org/x/time v0.2.0 - golang.org/x/tools v0.3.0 // indirect - google.golang.org/grpc v1.51.0 - google.golang.org/protobuf v1.28.1 + golang.org/x/crypto v0.9.0 + golang.org/x/net v0.10.0 // indirect + golang.org/x/sync v0.2.0 + golang.org/x/sys v0.8.0 // indirect + golang.org/x/time v0.3.0 + golang.org/x/tools v0.9.3 // indirect + google.golang.org/grpc v1.55.0 + google.golang.org/protobuf v1.30.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 @@ -69,8 +69,8 @@ require ( require ( github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b github.com/holiman/bloomfilter/v2 v2.0.3 - github.com/ledgerwatch/erigon-lib v0.0.0-20221218022306-0f8fdd40c2db - github.com/ledgerwatch/log/v3 v3.6.0 + github.com/ledgerwatch/erigon-lib v0.0.0-20230607152933-42c9c28cac68 + github.com/ledgerwatch/log/v3 v3.8.0 github.com/libp2p/go-libp2p-core v0.20.1 github.com/olekukonko/tablewriter v0.0.5 ) @@ -80,10 +80,10 @@ require ( github.com/BurntSushi/toml v1.2.0 // indirect github.com/DataDog/zstd v1.5.2 // indirect github.com/OpenPeeDeeP/depguard v1.0.1 // indirect - github.com/VictoriaMetrics/metrics v1.23.0 // indirect + github.com/VictoriaMetrics/metrics v1.23.1 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.2.2 // indirect + github.com/bits-and-blooms/bitset v1.5.0 // indirect github.com/bombsimon/wsl/v2 v2.0.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect @@ -96,7 +96,7 @@ require ( github.com/containerd/cgroups v1.0.4 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect - github.com/deckarep/golang-set/v2 v2.1.0 // indirect + github.com/deckarep/golang-set/v2 v2.3.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/dgraph-io/badger v1.6.2 // indirect github.com/dgraph-io/ristretto v0.0.3 // indirect @@ -147,12 +147,12 @@ require ( github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect github.com/google/uuid v1.3.0 // indirect github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 // indirect - github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/big v0.0.0-20221017200358-a027dc42d04e // indirect - github.com/holiman/uint256 v1.2.1 // indirect + github.com/holiman/uint256 v1.2.2 // indirect github.com/huin/goupnp v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/ipfs/go-cid v0.3.2 // indirect @@ -172,6 +172,7 @@ require ( github.com/koron/go-ssdp v0.0.3 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect + github.com/ledgerwatch/interfaces v0.0.0-20230602104541-cdc6e215fb3e // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect @@ -190,8 +191,9 @@ require ( github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb // indirect + github.com/matryer/moq v0.3.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-pointer v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect @@ -229,6 +231,7 @@ require ( github.com/prometheus/common v0.41.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/prometheus/tsdb v0.10.0 // indirect + github.com/quasilyte/go-ruleguard/dsl v0.3.22 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect @@ -249,7 +252,7 @@ require ( github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/tommy-muehle/go-mnd v1.1.1 // indirect - github.com/torquem-ch/mdbx-go v0.27.0 // indirect + github.com/torquem-ch/mdbx-go v0.27.10 // indirect github.com/tyler-smith/go-bip39 v1.1.0 // indirect github.com/ultraware/funlen v0.0.2 // indirect github.com/ultraware/whitespace v0.0.4 // indirect @@ -263,11 +266,12 @@ require ( go.uber.org/dig v1.15.0 // indirect go.uber.org/fx v1.18.2 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 // indirect - golang.org/x/mod v0.7.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect - google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e // indirect + golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect + golang.org/x/mod v0.10.0 // indirect + golang.org/x/term v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect + google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 8b86a286c2..51be04743c 100644 --- a/go.sum +++ b/go.sum @@ -90,6 +90,8 @@ github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bw github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= github.com/VictoriaMetrics/metrics v1.23.0 h1:WzfqyzCaxUZip+OBbg1+lV33WChDSu4ssYII3nxtpeA= github.com/VictoriaMetrics/metrics v1.23.0/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc= +github.com/VictoriaMetrics/metrics v1.23.1 h1:/j8DzeJBxSpL2qSIdqnRFLvQQhbJyJbbEi22yMm7oL0= +github.com/VictoriaMetrics/metrics v1.23.1/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc= github.com/Workiva/go-datastructures v1.0.50 h1:slDmfW6KCHcC7U+LP3DDBbm4fqTwZGn1beOFPfGaLvo= github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= github.com/Zilliqa/gozilliqa-sdk v1.2.1-0.20201201074141-dd0ecada1be6/go.mod h1:eSYp2T6f0apnuW8TzhV3f6Aff2SE8Dwio++U4ha4yEM= @@ -140,6 +142,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.2.2 h1:J5gbX05GpMdBjCvQ9MteIg2KKDExr7DrgK+Yc15FvIk= github.com/bits-and-blooms/bitset v1.2.2/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= +github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= @@ -185,6 +189,7 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -259,6 +264,8 @@ github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsP github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI= github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/deckarep/golang-set/v2 v2.3.0 h1:qs18EKUfHm2X9fA50Mr/M5hccg2tNnVqsiBImnyDs0g= +github.com/deckarep/golang-set/v2 v2.3.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= @@ -498,6 +505,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -615,6 +624,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmg github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= @@ -666,6 +677,8 @@ github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iU github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/holiman/uint256 v1.2.1 h1:XRtyuda/zw2l+Bq/38n5XUoEF72aSOu/77Thd9pPp2o= github.com/holiman/uint256 v1.2.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= +github.com/holiman/uint256 v1.2.2 h1:TXKcSGc2WaxPD2+bmzAsVthL4+pEN0YwXcL5qED83vk= +github.com/holiman/uint256 v1.2.2/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= @@ -838,8 +851,14 @@ github.com/labstack/gommon v0.3.1/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/ledgerwatch/erigon-lib v0.0.0-20221218022306-0f8fdd40c2db h1:wV9YkkYQArbUdTdlPxXi5BW6H9ovYbyUT8Af7foetvQ= github.com/ledgerwatch/erigon-lib v0.0.0-20221218022306-0f8fdd40c2db/go.mod h1:5GCPOzxAshLF7f0wrMZu2Bdq0qqIiMcIubM9n+25gGo= +github.com/ledgerwatch/erigon-lib v0.0.0-20230607152933-42c9c28cac68 h1:eKTHu42qcMwDR2qi2KfR3cKGU6Qli7ilPwqQDsT5XL0= +github.com/ledgerwatch/erigon-lib v0.0.0-20230607152933-42c9c28cac68/go.mod h1:gV87KL7+CmJ31bLSk0FbCnowHYi6w7kF8Q1vyiwttqg= +github.com/ledgerwatch/interfaces v0.0.0-20230602104541-cdc6e215fb3e h1:2tltVQCyMEk6Az7uSNRAt4S0+2rV4VJ4PCHK1f1rung= +github.com/ledgerwatch/interfaces v0.0.0-20230602104541-cdc6e215fb3e/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc= github.com/ledgerwatch/log/v3 v3.6.0 h1:JBUSK1epPyutUrz7KYDTcJtQLEHnehECRpKbM1ugy5M= github.com/ledgerwatch/log/v3 v3.6.0/go.mod h1:L+Sp+ma/h205EdCjviZECjGEvYUYEyXSdiuHNZzg+xQ= +github.com/ledgerwatch/log/v3 v3.8.0 h1:gCpp7uGtIerEz1jKVPeDnbIopFPud9ZnCpBLlLBGqPU= +github.com/ledgerwatch/log/v3 v3.8.0/go.mod h1:J2Jl6zV/58LeA6LTaVVnCGyf1/cYYSEOOLHY4ZN8S2A= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -895,6 +914,8 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/marten-seemann/qpack v0.3.0 h1:UiWstOgT8+znlkDPOg2+3rIuYXJ2CnGDkGUXN6ki6hE= +github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= +github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= github.com/marten-seemann/qtls-go1-18 v0.1.3 h1:R4H2Ks8P6pAtUagjFty2p7BVHn3XiwDAl7TTQf5h7TI= github.com/marten-seemann/qtls-go1-18 v0.1.3/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= github.com/marten-seemann/qtls-go1-19 v0.1.1 h1:mnbxeq3oEyQxQXwI4ReCgW9DPoPR94sNlqWoDZnjRIE= @@ -905,6 +926,8 @@ github.com/marten-seemann/webtransport-go v0.4.1 h1:8Ir7OoAvtp79yxQpa3foTKIPuoH+ github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE= github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= +github.com/matryer/moq v0.3.1 h1:kLDiBJoGcusWS2BixGyTkF224aSCD8nLY24tj/NcTCs= +github.com/matryer/moq v0.3.1/go.mod h1:RJ75ZZZD71hejp39j4crZLsEDszGk6iH4v4YsWFKH4s= github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -925,6 +948,8 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -1172,6 +1197,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= +github.com/quasilyte/go-ruleguard/dsl v0.3.22 h1:wd8zkOhSNr+I+8Qeciml08ivDt1pSXe60+5DqOpCjPE= +github.com/quasilyte/go-ruleguard/dsl v0.3.22/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= @@ -1319,6 +1346,8 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= @@ -1355,6 +1384,8 @@ github.com/tommy-muehle/go-mnd v1.1.1 h1:4D0wuPKjOTiK2garzuPGGvm4zZ/wLYDOH8TJSAB github.com/tommy-muehle/go-mnd v1.1.1/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig= github.com/torquem-ch/mdbx-go v0.27.0 h1:FquhRvKL2zweMdk1R6UdOx3h6DiHgJ0+P9yQvSouURI= github.com/torquem-ch/mdbx-go v0.27.0/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E= +github.com/torquem-ch/mdbx-go v0.27.10 h1:iwb8Wn9gse4MEYIltAna+pxMPCY7hA1/5LLN/Qrcsx0= +github.com/torquem-ch/mdbx-go v0.27.10/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E= github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= @@ -1398,6 +1429,7 @@ github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= +github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= @@ -1480,6 +1512,7 @@ go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= @@ -1526,6 +1559,8 @@ golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1544,6 +1579,8 @@ golang.org/x/exp v0.0.0-20220426173459-3bcf042a4bf5/go.mod h1:lgLbSvA5ygNOMpwM/9 golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 h1:Jvc7gsqn21cJHCmAWx0LiimpP18LZmUxkT5Mp7EZ1mI= golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1577,6 +1614,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1654,6 +1693,8 @@ golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1687,6 +1728,8 @@ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1779,6 +1822,7 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1803,6 +1847,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1810,6 +1856,8 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1823,6 +1871,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1834,6 +1884,8 @@ golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE= golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1923,6 +1975,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= +golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2024,6 +2078,8 @@ google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxH google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e h1:S9GbmC1iCgvbLyAokVCwiO6tVIrU9Y7c5oMx1V/ki/Y= google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= @@ -2053,6 +2109,10 @@ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 h1:rNBFJjBCOgVr9pWD7rs/knKL4FRTKgpZmsRfV214zcA= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -2069,6 +2129,8 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From e4088f9e83e78af2201da68d239ace77b927254b Mon Sep 17 00:00:00 2001 From: Konstantin <355847+Frozen@users.noreply.github.com> Date: Sat, 10 Jun 2023 11:05:51 -0400 Subject: [PATCH 417/420] Fixed bug with storing already cancelled context. (#4445) * Fixed context usages. * Additional fixes. * Increased timeout. * Additional logs. * Fixed short range. --- .../stagedstreamsync/short_range_helper.go | 44 +++++------ api/service/stagedstreamsync/stage.go | 9 +-- api/service/stagedstreamsync/stage_bodies.go | 77 ++++++++----------- api/service/stagedstreamsync/stage_epoch.go | 39 ++++------ api/service/stagedstreamsync/stage_finish.go | 28 +++---- api/service/stagedstreamsync/stage_heads.go | 31 +++----- .../stagedstreamsync/stage_short_range.go | 42 ++++------ api/service/stagedstreamsync/stage_state.go | 36 +++------ .../stagedstreamsync/staged_stream_sync.go | 59 ++++++-------- api/service/stagedstreamsync/syncing.go | 30 ++++---- consensus/consensus_service.go | 2 +- 11 files changed, 162 insertions(+), 235 deletions(-) diff --git a/api/service/stagedstreamsync/short_range_helper.go b/api/service/stagedstreamsync/short_range_helper.go index 90415c87c3..e43b3c6916 100644 --- a/api/service/stagedstreamsync/short_range_helper.go +++ b/api/service/stagedstreamsync/short_range_helper.go @@ -15,13 +15,11 @@ import ( type srHelper struct { syncProtocol syncProtocol - - ctx context.Context - config Config - logger zerolog.Logger + config Config + logger zerolog.Logger } -func (sh *srHelper) getHashChain(bns []uint64) ([]common.Hash, []sttypes.StreamID, error) { +func (sh *srHelper) getHashChain(ctx context.Context, bns []uint64) ([]common.Hash, []sttypes.StreamID, error) { results := newBlockHashResults(bns) var wg sync.WaitGroup @@ -31,7 +29,7 @@ func (sh *srHelper) getHashChain(bns []uint64) ([]common.Hash, []sttypes.StreamI go func(index int) { defer wg.Done() - hashes, stid, err := sh.doGetBlockHashesRequest(bns) + hashes, stid, err := sh.doGetBlockHashesRequest(ctx, bns) if err != nil { sh.logger.Warn().Err(err).Str("StreamID", string(stid)). Msg(WrapStagedSyncMsg("doGetBlockHashes return error")) @@ -43,10 +41,10 @@ func (sh *srHelper) getHashChain(bns []uint64) ([]common.Hash, []sttypes.StreamI wg.Wait() select { - case <-sh.ctx.Done(): - sh.logger.Info().Err(sh.ctx.Err()).Int("num blocks", results.numBlocksWithResults()). + case <-ctx.Done(): + sh.logger.Info().Err(ctx.Err()).Int("num blocks", results.numBlocksWithResults()). Msg(WrapStagedSyncMsg("short range sync get hashes timed out")) - return nil, nil, sh.ctx.Err() + return nil, nil, ctx.Err() default: } @@ -56,13 +54,12 @@ func (sh *srHelper) getHashChain(bns []uint64) ([]common.Hash, []sttypes.StreamI return hashChain, wl, nil } -func (sh *srHelper) getBlocksChain(bns []uint64) ([]*types.Block, sttypes.StreamID, error) { - return sh.doGetBlocksByNumbersRequest(bns) +func (sh *srHelper) getBlocksChain(ctx context.Context, bns []uint64) ([]*types.Block, sttypes.StreamID, error) { + return sh.doGetBlocksByNumbersRequest(ctx, bns) } -func (sh *srHelper) getBlocksByHashes(hashes []common.Hash, whitelist []sttypes.StreamID) ([]*types.Block, []sttypes.StreamID, error) { - ctx, cancel := context.WithCancel(sh.ctx) - defer cancel() +func (sh *srHelper) getBlocksByHashes(ctx context.Context, hashes []common.Hash, whitelist []sttypes.StreamID) ([]*types.Block, []sttypes.StreamID, error) { + m := newGetBlocksByHashManager(hashes, whitelist) var ( @@ -80,7 +77,8 @@ func (sh *srHelper) getBlocksByHashes(hashes []common.Hash, whitelist []sttypes. for i := 0; i != concurrency; i++ { go func(index int) { defer wg.Done() - defer cancel() // it's ok to cancel context more than once + ctx, cancel := context.WithCancel(ctx) + defer cancel() for { if m.isDone() { @@ -121,11 +119,11 @@ func (sh *srHelper) getBlocksByHashes(hashes []common.Hash, whitelist []sttypes. return nil, nil, gErr } select { - case <-sh.ctx.Done(): + case <-ctx.Done(): res, _, _ := m.getResults() - sh.logger.Info().Err(sh.ctx.Err()).Int("num blocks", len(res)). + sh.logger.Info().Err(ctx.Err()).Int("num blocks", len(res)). Msg(WrapStagedSyncMsg("short range sync get blocks timed out")) - return nil, nil, sh.ctx.Err() + return nil, nil, ctx.Err() default: } @@ -149,8 +147,8 @@ func (sh *srHelper) prepareBlockHashNumbers(curNumber uint64) []uint64 { return res } -func (sh *srHelper) doGetBlockHashesRequest(bns []uint64) ([]common.Hash, sttypes.StreamID, error) { - ctx, cancel := context.WithTimeout(sh.ctx, 1*time.Second) +func (sh *srHelper) doGetBlockHashesRequest(ctx context.Context, bns []uint64) ([]common.Hash, sttypes.StreamID, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() hashes, stid, err := sh.syncProtocol.GetBlockHashes(ctx, bns) @@ -171,8 +169,8 @@ func (sh *srHelper) doGetBlockHashesRequest(bns []uint64) ([]common.Hash, sttype return hashes, stid, nil } -func (sh *srHelper) doGetBlocksByNumbersRequest(bns []uint64) ([]*types.Block, sttypes.StreamID, error) { - ctx, cancel := context.WithTimeout(sh.ctx, 10*time.Second) +func (sh *srHelper) doGetBlocksByNumbersRequest(ctx context.Context, bns []uint64) ([]*types.Block, sttypes.StreamID, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() blocks, stid, err := sh.syncProtocol.GetBlocksByNumber(ctx, bns) @@ -186,7 +184,7 @@ func (sh *srHelper) doGetBlocksByNumbersRequest(bns []uint64) ([]*types.Block, s } func (sh *srHelper) doGetBlocksByHashesRequest(ctx context.Context, hashes []common.Hash, wl []sttypes.StreamID) ([]*types.Block, sttypes.StreamID, error) { - ctx, cancel := context.WithTimeout(sh.ctx, 10*time.Second) + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() blocks, stid, err := sh.syncProtocol.GetBlocksByHashes(ctx, hashes, diff --git a/api/service/stagedstreamsync/stage.go b/api/service/stagedstreamsync/stage.go index 255560a0ff..48334a5e52 100644 --- a/api/service/stagedstreamsync/stage.go +++ b/api/service/stagedstreamsync/stage.go @@ -16,21 +16,18 @@ type StageHandler interface { // * invalidBlockRevert - whether the execution is to solve the invalid block // * s - is the current state of the stage and contains stage data. // * reverter - if the stage needs to cause reverting, `reverter` methods can be used. - Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error + Exec(ctx context.Context, firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error // Revert is the reverting logic of the stage. // * firstCycle - is it the first cycle of syncing. // * u - contains information about the revert itself. // * s - represents the state of this stage at the beginning of revert. - Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) error + Revert(ctx context.Context, firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) error // CleanUp is the execution function for the stage to prune old data. // * firstCycle - is it the first cycle of syncing. // * p - is the current state of the stage and contains stage data. - CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) error - - // SetStageContext updates the context for stage - SetStageContext(ctx context.Context) + CleanUp(ctx context.Context, firstCycle bool, p *CleanUpState, tx kv.RwTx) error } // Stage is a single sync stage in staged sync. diff --git a/api/service/stagedstreamsync/stage_bodies.go b/api/service/stagedstreamsync/stage_bodies.go index cb06e98e8a..62309b76df 100644 --- a/api/service/stagedstreamsync/stage_bodies.go +++ b/api/service/stagedstreamsync/stage_bodies.go @@ -17,8 +17,8 @@ import ( type StageBodies struct { configs StageBodiesCfg } + type StageBodiesCfg struct { - ctx context.Context bc core.BlockChain db kv.RwDB blockDBs []kv.RwDB @@ -34,9 +34,8 @@ func NewStageBodies(cfg StageBodiesCfg) *StageBodies { } } -func NewStageBodiesCfg(ctx context.Context, bc core.BlockChain, db kv.RwDB, blockDBs []kv.RwDB, concurrency int, protocol syncProtocol, isBeacon bool, logProgress bool) StageBodiesCfg { +func NewStageBodiesCfg(bc core.BlockChain, db kv.RwDB, blockDBs []kv.RwDB, concurrency int, protocol syncProtocol, isBeacon bool, logProgress bool) StageBodiesCfg { return StageBodiesCfg{ - ctx: ctx, bc: bc, db: db, blockDBs: blockDBs, @@ -47,17 +46,13 @@ func NewStageBodiesCfg(ctx context.Context, bc core.BlockChain, db kv.RwDB, bloc } } -func (b *StageBodies) SetStageContext(ctx context.Context) { - b.configs.ctx = ctx -} - // Exec progresses Bodies stage in the forward direction -func (b *StageBodies) Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) (err error) { +func (b *StageBodies) Exec(ctx context.Context, firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) (err error) { useInternalTx := tx == nil if invalidBlockRevert { - return b.redownloadBadBlock(s) + return b.redownloadBadBlock(ctx, s) } // for short range sync, skip this stage @@ -72,10 +67,8 @@ func (b *StageBodies) Exec(firstCycle bool, invalidBlockRevert bool, s *StageSta } currProgress := uint64(0) targetHeight := s.state.currentCycle.TargetHeight - // isBeacon := s.state.isBeacon - // isLastCycle := targetHeight >= maxHeight - if errV := CreateView(b.configs.ctx, b.configs.db, tx, func(etx kv.Tx) error { + if errV := CreateView(ctx, b.configs.db, tx, func(etx kv.Tx) error { if currProgress, err = s.CurrentStageProgress(etx); err != nil { return err } @@ -85,7 +78,7 @@ func (b *StageBodies) Exec(firstCycle bool, invalidBlockRevert bool, s *StageSta } if currProgress == 0 { - if err := b.cleanAllBlockDBs(); err != nil { + if err := b.cleanAllBlockDBs(ctx); err != nil { return err } currProgress = currentHead @@ -104,7 +97,7 @@ func (b *StageBodies) Exec(firstCycle bool, invalidBlockRevert bool, s *StageSta if useInternalTx { var err error - tx, err = b.configs.db.BeginRw(context.Background()) + tx, err = b.configs.db.BeginRw(ctx) if err != nil { return err } @@ -119,7 +112,7 @@ func (b *StageBodies) Exec(firstCycle bool, invalidBlockRevert bool, s *StageSta for i := 0; i != s.state.config.Concurrency; i++ { wg.Add(1) - go b.runBlockWorkerLoop(s.state.gbm, &wg, i, startTime) + go b.runBlockWorkerLoop(ctx, s.state.gbm, &wg, i, startTime) } wg.Wait() @@ -134,7 +127,7 @@ func (b *StageBodies) Exec(firstCycle bool, invalidBlockRevert bool, s *StageSta } // runBlockWorkerLoop creates a work loop for download blocks -func (b *StageBodies) runBlockWorkerLoop(gbm *blockDownloadManager, wg *sync.WaitGroup, loopID int, startTime time.Time) { +func (b *StageBodies) runBlockWorkerLoop(ctx context.Context, gbm *blockDownloadManager, wg *sync.WaitGroup, loopID int, startTime time.Time) { currentBlock := int(b.configs.bc.CurrentBlock().NumberU64()) @@ -142,21 +135,21 @@ func (b *StageBodies) runBlockWorkerLoop(gbm *blockDownloadManager, wg *sync.Wai for { select { - case <-b.configs.ctx.Done(): + case <-ctx.Done(): return default: } batch := gbm.GetNextBatch() if len(batch) == 0 { select { - case <-b.configs.ctx.Done(): + case <-ctx.Done(): return case <-time.After(100 * time.Millisecond): return } } - blockBytes, sigBytes, stid, err := b.downloadRawBlocks(batch) + blockBytes, sigBytes, stid, err := b.downloadRawBlocks(ctx, batch) if err != nil { if !errors.Is(err, context.Canceled) { b.configs.protocol.StreamFailed(stid, "downloadRawBlocks failed") @@ -176,7 +169,7 @@ func (b *StageBodies) runBlockWorkerLoop(gbm *blockDownloadManager, wg *sync.Wai err := errors.New("downloadRawBlocks received empty blockBytes") gbm.HandleRequestError(batch, err, stid) } else { - if err = b.saveBlocks(gbm.tx, batch, blockBytes, sigBytes, loopID, stid); err != nil { + if err = b.saveBlocks(ctx, gbm.tx, batch, blockBytes, sigBytes, loopID, stid); err != nil { panic(ErrSaveBlocksToDbFailed) } gbm.HandleRequestResult(batch, blockBytes, sigBytes, loopID, stid) @@ -197,7 +190,7 @@ func (b *StageBodies) runBlockWorkerLoop(gbm *blockDownloadManager, wg *sync.Wai } // redownloadBadBlock tries to redownload the bad block from other streams -func (b *StageBodies) redownloadBadBlock(s *StageState) error { +func (b *StageBodies) redownloadBadBlock(ctx context.Context, s *StageState) error { batch := make([]uint64, 1) batch = append(batch, s.state.invalidBlock.Number) @@ -206,7 +199,7 @@ func (b *StageBodies) redownloadBadBlock(s *StageState) error { if b.configs.protocol.NumStreams() == 0 { return errors.Errorf("re-download bad block from all streams failed") } - blockBytes, sigBytes, stid, err := b.downloadRawBlocks(batch) + blockBytes, sigBytes, stid, err := b.downloadRawBlocks(ctx, batch) if err != nil { if !errors.Is(err, context.Canceled) { b.configs.protocol.StreamFailed(stid, "tried to re-download bad block from this stream, but downloadRawBlocks failed") @@ -225,8 +218,8 @@ func (b *StageBodies) redownloadBadBlock(s *StageState) error { continue } s.state.gbm.SetDownloadDetails(batch, 0, stid) - if errU := b.configs.blockDBs[0].Update(context.Background(), func(tx kv.RwTx) error { - if err = b.saveBlocks(tx, batch, blockBytes, sigBytes, 0, stid); err != nil { + if errU := b.configs.blockDBs[0].Update(ctx, func(tx kv.RwTx) error { + if err = b.saveBlocks(ctx, tx, batch, blockBytes, sigBytes, 0, stid); err != nil { return errors.Errorf("[STAGED_STREAM_SYNC] saving re-downloaded bad block to db failed.") } return nil @@ -238,8 +231,8 @@ func (b *StageBodies) redownloadBadBlock(s *StageState) error { return nil } -func (b *StageBodies) downloadBlocks(bns []uint64) ([]*types.Block, sttypes.StreamID, error) { - ctx, cancel := context.WithTimeout(b.configs.ctx, 10*time.Second) +func (b *StageBodies) downloadBlocks(ctx context.Context, bns []uint64) ([]*types.Block, sttypes.StreamID, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() blocks, stid, err := b.configs.protocol.GetBlocksByNumber(ctx, bns) @@ -252,8 +245,8 @@ func (b *StageBodies) downloadBlocks(bns []uint64) ([]*types.Block, sttypes.Stre return blocks, stid, nil } -func (b *StageBodies) downloadRawBlocks(bns []uint64) ([][]byte, [][]byte, sttypes.StreamID, error) { - ctx, cancel := context.WithTimeout(b.configs.ctx, 10*time.Second) +func (b *StageBodies) downloadRawBlocks(ctx context.Context, bns []uint64) ([][]byte, [][]byte, sttypes.StreamID, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() return b.configs.protocol.GetRawBlocksByNumber(ctx, bns) @@ -272,9 +265,9 @@ func validateGetBlocksResult(requested []uint64, result []*types.Block) error { } // saveBlocks saves the blocks into db -func (b *StageBodies) saveBlocks(tx kv.RwTx, bns []uint64, blockBytes [][]byte, sigBytes [][]byte, loopID int, stid sttypes.StreamID) error { +func (b *StageBodies) saveBlocks(ctx context.Context, tx kv.RwTx, bns []uint64, blockBytes [][]byte, sigBytes [][]byte, loopID int, stid sttypes.StreamID) error { - tx, err := b.configs.blockDBs[loopID].BeginRw(context.Background()) + tx, err := b.configs.blockDBs[loopID].BeginRw(ctx) if err != nil { return err } @@ -313,11 +306,11 @@ func (b *StageBodies) saveBlocks(tx kv.RwTx, bns []uint64, blockBytes [][]byte, return nil } -func (b *StageBodies) saveProgress(s *StageState, progress uint64, tx kv.RwTx) (err error) { +func (b *StageBodies) saveProgress(ctx context.Context, s *StageState, progress uint64, tx kv.RwTx) (err error) { useInternalTx := tx == nil if useInternalTx { var err error - tx, err = b.configs.db.BeginRw(context.Background()) + tx, err = b.configs.db.BeginRw(ctx) if err != nil { return err } @@ -340,9 +333,8 @@ func (b *StageBodies) saveProgress(s *StageState, progress uint64, tx kv.RwTx) ( return nil } -func (b *StageBodies) cleanBlocksDB(loopID int) (err error) { - - tx, errb := b.configs.blockDBs[loopID].BeginRw(b.configs.ctx) +func (b *StageBodies) cleanBlocksDB(ctx context.Context, loopID int) (err error) { + tx, errb := b.configs.blockDBs[loopID].BeginRw(ctx) if errb != nil { return errb } @@ -370,26 +362,26 @@ func (b *StageBodies) cleanBlocksDB(loopID int) (err error) { return nil } -func (b *StageBodies) cleanAllBlockDBs() (err error) { +func (b *StageBodies) cleanAllBlockDBs(ctx context.Context) (err error) { //clean all blocks DBs for i := 0; i < b.configs.concurrency; i++ { - if err := b.cleanBlocksDB(i); err != nil { + if err := b.cleanBlocksDB(ctx, i); err != nil { return err } } return nil } -func (b *StageBodies) Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { +func (b *StageBodies) Revert(ctx context.Context, firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { //clean all blocks DBs - if err := b.cleanAllBlockDBs(); err != nil { + if err := b.cleanAllBlockDBs(ctx); err != nil { return err } useInternalTx := tx == nil if useInternalTx { - tx, err = b.configs.db.BeginRw(b.configs.ctx) + tx, err = b.configs.db.BeginRw(ctx) if err != nil { return err } @@ -416,10 +408,9 @@ func (b *StageBodies) Revert(firstCycle bool, u *RevertState, s *StageState, tx return nil } -func (b *StageBodies) CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { - +func (b *StageBodies) CleanUp(ctx context.Context, firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { //clean all blocks DBs - if err := b.cleanAllBlockDBs(); err != nil { + if err := b.cleanAllBlockDBs(ctx); err != nil { return err } diff --git a/api/service/stagedstreamsync/stage_epoch.go b/api/service/stagedstreamsync/stage_epoch.go index 77dc57bfdd..394a1d5d69 100644 --- a/api/service/stagedstreamsync/stage_epoch.go +++ b/api/service/stagedstreamsync/stage_epoch.go @@ -15,9 +15,8 @@ type StageEpoch struct { } type StageEpochCfg struct { - ctx context.Context - bc core.BlockChain - db kv.RwDB + bc core.BlockChain + db kv.RwDB } func NewStageEpoch(cfg StageEpochCfg) *StageEpoch { @@ -26,19 +25,14 @@ func NewStageEpoch(cfg StageEpochCfg) *StageEpoch { } } -func NewStageEpochCfg(ctx context.Context, bc core.BlockChain, db kv.RwDB) StageEpochCfg { +func NewStageEpochCfg(bc core.BlockChain, db kv.RwDB) StageEpochCfg { return StageEpochCfg{ - ctx: ctx, - bc: bc, - db: db, + bc: bc, + db: db, } } -func (sr *StageEpoch) SetStageContext(ctx context.Context) { - sr.configs.ctx = ctx -} - -func (sr *StageEpoch) Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error { +func (sr *StageEpoch) Exec(ctx context.Context, firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error { // no need to update epoch chain if we are redoing the stages because of bad block if invalidBlockRevert { @@ -54,7 +48,7 @@ func (sr *StageEpoch) Exec(firstCycle bool, invalidBlockRevert bool, s *StageSta } // doShortRangeSyncForEpochSync - n, err := sr.doShortRangeSyncForEpochSync(s) + n, err := sr.doShortRangeSyncForEpochSync(ctx, s) s.state.inserted = n if err != nil { return err @@ -63,7 +57,7 @@ func (sr *StageEpoch) Exec(firstCycle bool, invalidBlockRevert bool, s *StageSta useInternalTx := tx == nil if useInternalTx { var err error - tx, err = sr.configs.db.BeginRw(sr.configs.ctx) + tx, err = sr.configs.db.BeginRw(ctx) if err != nil { return err } @@ -79,17 +73,16 @@ func (sr *StageEpoch) Exec(firstCycle bool, invalidBlockRevert bool, s *StageSta return nil } -func (sr *StageEpoch) doShortRangeSyncForEpochSync(s *StageState) (int, error) { +func (sr *StageEpoch) doShortRangeSyncForEpochSync(ctx context.Context, s *StageState) (int, error) { numShortRangeCounterVec.With(s.state.promLabels()).Inc() - srCtx, cancel := context.WithTimeout(s.state.ctx, ShortRangeTimeout) + ctx, cancel := context.WithTimeout(ctx, ShortRangeTimeout) defer cancel() //TODO: merge srHelper with StageEpochConfig sh := &srHelper{ syncProtocol: s.state.protocol, - ctx: srCtx, config: s.state.config, logger: utils.Logger().With().Str("mode", "epoch chain short range").Logger(), } @@ -116,7 +109,7 @@ func (sr *StageEpoch) doShortRangeSyncForEpochSync(s *StageState) (int, error) { } //////////////////////////////////////////////////////// - hashChain, whitelist, err := sh.getHashChain(bns) + hashChain, whitelist, err := sh.getHashChain(ctx, bns) if err != nil { return 0, errors.Wrap(err, "getHashChain") } @@ -124,7 +117,7 @@ func (sr *StageEpoch) doShortRangeSyncForEpochSync(s *StageState) (int, error) { // short circuit for no sync is needed return 0, nil } - blocks, streamID, err := sh.getBlocksByHashes(hashChain, whitelist) + blocks, streamID, err := sh.getBlocksByHashes(ctx, hashChain, whitelist) if err != nil { utils.Logger().Warn().Err(err).Msg("epoch sync getBlocksByHashes failed") if !errors.Is(err, context.Canceled) { @@ -157,10 +150,10 @@ func (sr *StageEpoch) doShortRangeSyncForEpochSync(s *StageState) (int, error) { return n, nil } -func (sr *StageEpoch) Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { +func (sr *StageEpoch) Revert(ctx context.Context, firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { useInternalTx := tx == nil if useInternalTx { - tx, err = sr.configs.db.BeginRw(context.Background()) + tx, err = sr.configs.db.BeginRw(ctx) if err != nil { return err } @@ -179,10 +172,10 @@ func (sr *StageEpoch) Revert(firstCycle bool, u *RevertState, s *StageState, tx return nil } -func (sr *StageEpoch) CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { +func (sr *StageEpoch) CleanUp(ctx context.Context, firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { useInternalTx := tx == nil if useInternalTx { - tx, err = sr.configs.db.BeginRw(context.Background()) + tx, err = sr.configs.db.BeginRw(ctx) if err != nil { return err } diff --git a/api/service/stagedstreamsync/stage_finish.go b/api/service/stagedstreamsync/stage_finish.go index 9039b5588d..0dfae53ae2 100644 --- a/api/service/stagedstreamsync/stage_finish.go +++ b/api/service/stagedstreamsync/stage_finish.go @@ -11,8 +11,7 @@ type StageFinish struct { } type StageFinishCfg struct { - ctx context.Context - db kv.RwDB + db kv.RwDB } func NewStageFinish(cfg StageFinishCfg) *StageFinish { @@ -21,22 +20,17 @@ func NewStageFinish(cfg StageFinishCfg) *StageFinish { } } -func NewStageFinishCfg(ctx context.Context, db kv.RwDB) StageFinishCfg { +func NewStageFinishCfg(db kv.RwDB) StageFinishCfg { return StageFinishCfg{ - ctx: ctx, - db: db, + db: db, } } -func (finish *StageFinish) SetStageContext(ctx context.Context) { - finish.configs.ctx = ctx -} - -func (finish *StageFinish) Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error { +func (finish *StageFinish) Exec(ctx context.Context, firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error { useInternalTx := tx == nil if useInternalTx { var err error - tx, err = finish.configs.db.BeginRw(context.Background()) + tx, err = finish.configs.db.BeginRw(ctx) if err != nil { return err } @@ -54,11 +48,11 @@ func (finish *StageFinish) Exec(firstCycle bool, invalidBlockRevert bool, s *Sta return nil } -func (bh *StageFinish) clearBucket(tx kv.RwTx, isBeacon bool) error { +func (finish *StageFinish) clearBucket(ctx context.Context, tx kv.RwTx, isBeacon bool) error { useInternalTx := tx == nil if useInternalTx { var err error - tx, err = bh.configs.db.BeginRw(context.Background()) + tx, err = finish.configs.db.BeginRw(ctx) if err != nil { return err } @@ -73,10 +67,10 @@ func (bh *StageFinish) clearBucket(tx kv.RwTx, isBeacon bool) error { return nil } -func (finish *StageFinish) Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { +func (finish *StageFinish) Revert(ctx context.Context, firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { useInternalTx := tx == nil if useInternalTx { - tx, err = finish.configs.db.BeginRw(finish.configs.ctx) + tx, err = finish.configs.db.BeginRw(ctx) if err != nil { return err } @@ -95,10 +89,10 @@ func (finish *StageFinish) Revert(firstCycle bool, u *RevertState, s *StageState return nil } -func (finish *StageFinish) CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { +func (finish *StageFinish) CleanUp(ctx context.Context, firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { useInternalTx := tx == nil if useInternalTx { - tx, err = finish.configs.db.BeginRw(finish.configs.ctx) + tx, err = finish.configs.db.BeginRw(ctx) if err != nil { return err } diff --git a/api/service/stagedstreamsync/stage_heads.go b/api/service/stagedstreamsync/stage_heads.go index 8e1531a5ee..c917884a36 100644 --- a/api/service/stagedstreamsync/stage_heads.go +++ b/api/service/stagedstreamsync/stage_heads.go @@ -13,9 +13,8 @@ type StageHeads struct { } type StageHeadsCfg struct { - ctx context.Context - bc core.BlockChain - db kv.RwDB + bc core.BlockChain + db kv.RwDB } func NewStageHeads(cfg StageHeadsCfg) *StageHeads { @@ -24,20 +23,14 @@ func NewStageHeads(cfg StageHeadsCfg) *StageHeads { } } -func NewStageHeadersCfg(ctx context.Context, bc core.BlockChain, db kv.RwDB) StageHeadsCfg { +func NewStageHeadersCfg(bc core.BlockChain, db kv.RwDB) StageHeadsCfg { return StageHeadsCfg{ - ctx: ctx, - bc: bc, - db: db, + bc: bc, + db: db, } } -func (heads *StageHeads) SetStageContext(ctx context.Context) { - heads.configs.ctx = ctx -} - -func (heads *StageHeads) Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error { - +func (heads *StageHeads) Exec(ctx context.Context, firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error { // no need to update target if we are redoing the stages because of bad block if invalidBlockRevert { return nil @@ -51,7 +44,7 @@ func (heads *StageHeads) Exec(firstCycle bool, invalidBlockRevert bool, s *Stage useInternalTx := tx == nil if useInternalTx { var err error - tx, err = heads.configs.db.BeginRw(heads.configs.ctx) + tx, err = heads.configs.db.BeginRw(ctx) if err != nil { return err } @@ -63,7 +56,7 @@ func (heads *StageHeads) Exec(firstCycle bool, invalidBlockRevert bool, s *Stage currentHeight := heads.configs.bc.CurrentBlock().NumberU64() s.state.currentCycle.TargetHeight = maxHeight targetHeight := uint64(0) - if errV := CreateView(heads.configs.ctx, heads.configs.db, tx, func(etx kv.Tx) (err error) { + if errV := CreateView(ctx, heads.configs.db, tx, func(etx kv.Tx) (err error) { if targetHeight, err = s.CurrentStageProgress(etx); err != nil { return err } @@ -114,10 +107,10 @@ func (heads *StageHeads) Exec(firstCycle bool, invalidBlockRevert bool, s *Stage return nil } -func (heads *StageHeads) Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { +func (heads *StageHeads) Revert(ctx context.Context, firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { useInternalTx := tx == nil if useInternalTx { - tx, err = heads.configs.db.BeginRw(context.Background()) + tx, err = heads.configs.db.BeginRw(ctx) if err != nil { return err } @@ -136,10 +129,10 @@ func (heads *StageHeads) Revert(firstCycle bool, u *RevertState, s *StageState, return nil } -func (heads *StageHeads) CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { +func (heads *StageHeads) CleanUp(ctx context.Context, firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { useInternalTx := tx == nil if useInternalTx { - tx, err = heads.configs.db.BeginRw(context.Background()) + tx, err = heads.configs.db.BeginRw(ctx) if err != nil { return err } diff --git a/api/service/stagedstreamsync/stage_short_range.go b/api/service/stagedstreamsync/stage_short_range.go index 75f51ee1e7..8fb2f3059e 100644 --- a/api/service/stagedstreamsync/stage_short_range.go +++ b/api/service/stagedstreamsync/stage_short_range.go @@ -16,9 +16,8 @@ type StageShortRange struct { } type StageShortRangeCfg struct { - ctx context.Context - bc core.BlockChain - db kv.RwDB + bc core.BlockChain + db kv.RwDB } func NewStageShortRange(cfg StageShortRangeCfg) *StageShortRange { @@ -27,20 +26,14 @@ func NewStageShortRange(cfg StageShortRangeCfg) *StageShortRange { } } -func NewStageShortRangeCfg(ctx context.Context, bc core.BlockChain, db kv.RwDB) StageShortRangeCfg { +func NewStageShortRangeCfg(bc core.BlockChain, db kv.RwDB) StageShortRangeCfg { return StageShortRangeCfg{ - ctx: ctx, - bc: bc, - db: db, + bc: bc, + db: db, } } -func (sr *StageShortRange) SetStageContext(ctx context.Context) { - sr.configs.ctx = ctx -} - -func (sr *StageShortRange) Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error { - +func (sr *StageShortRange) Exec(ctx context.Context, firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) error { // no need to do short range if we are redoing the stages because of bad block if invalidBlockRevert { return nil @@ -56,7 +49,7 @@ func (sr *StageShortRange) Exec(firstCycle bool, invalidBlockRevert bool, s *Sta } // do short range sync - n, err := sr.doShortRangeSync(s) + n, err := sr.doShortRangeSync(ctx, s) s.state.inserted = n if err != nil { return err @@ -65,7 +58,7 @@ func (sr *StageShortRange) Exec(firstCycle bool, invalidBlockRevert bool, s *Sta useInternalTx := tx == nil if useInternalTx { var err error - tx, err = sr.configs.db.BeginRw(sr.configs.ctx) + tx, err = sr.configs.db.BeginRw(ctx) if err != nil { return err } @@ -87,16 +80,13 @@ func (sr *StageShortRange) Exec(firstCycle bool, invalidBlockRevert bool, s *Sta // 1. Obtain the block hashes and compute the longest hash chain.. // 2. Get blocks by hashes from computed hash chain. // 3. Insert the blocks to blockchain. -func (sr *StageShortRange) doShortRangeSync(s *StageState) (int, error) { - +func (sr *StageShortRange) doShortRangeSync(ctx context.Context, s *StageState) (int, error) { numShortRangeCounterVec.With(s.state.promLabels()).Inc() - - srCtx, cancel := context.WithTimeout(s.state.ctx, ShortRangeTimeout) + ctx, cancel := context.WithTimeout(ctx, ShortRangeTimeout) defer cancel() sh := &srHelper{ syncProtocol: s.state.protocol, - ctx: srCtx, config: s.state.config, logger: utils.Logger().With().Str("mode", "short range").Logger(), } @@ -106,7 +96,7 @@ func (sr *StageShortRange) doShortRangeSync(s *StageState) (int, error) { } curBN := sr.configs.bc.CurrentBlock().NumberU64() blkNums := sh.prepareBlockHashNumbers(curBN) - hashChain, whitelist, err := sh.getHashChain(blkNums) + hashChain, whitelist, err := sh.getHashChain(ctx, blkNums) if err != nil { return 0, errors.Wrap(err, "getHashChain") } @@ -130,7 +120,7 @@ func (sr *StageShortRange) doShortRangeSync(s *StageState) (int, error) { s.state.status.finishSyncing() }() - blocks, stids, err := sh.getBlocksByHashes(hashChain, whitelist) + blocks, stids, err := sh.getBlocksByHashes(ctx, hashChain, whitelist) if err != nil { utils.Logger().Warn().Err(err).Msg("getBlocksByHashes failed") if !errors.Is(err, context.Canceled) { @@ -159,10 +149,10 @@ func (sr *StageShortRange) doShortRangeSync(s *StageState) (int, error) { return n, nil } -func (sr *StageShortRange) Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { +func (sr *StageShortRange) Revert(ctx context.Context, firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { useInternalTx := tx == nil if useInternalTx { - tx, err = sr.configs.db.BeginRw(context.Background()) + tx, err = sr.configs.db.BeginRw(ctx) if err != nil { return err } @@ -181,10 +171,10 @@ func (sr *StageShortRange) Revert(firstCycle bool, u *RevertState, s *StageState return nil } -func (sr *StageShortRange) CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { +func (sr *StageShortRange) CleanUp(ctx context.Context, firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { useInternalTx := tx == nil if useInternalTx { - tx, err = sr.configs.db.BeginRw(context.Background()) + tx, err = sr.configs.db.BeginRw(ctx) if err != nil { return err } diff --git a/api/service/stagedstreamsync/stage_state.go b/api/service/stagedstreamsync/stage_state.go index 9eda042477..4b237c2916 100644 --- a/api/service/stagedstreamsync/stage_state.go +++ b/api/service/stagedstreamsync/stage_state.go @@ -11,7 +11,6 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/utils" "github.com/ledgerwatch/erigon-lib/kv" - "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog" ) @@ -19,7 +18,6 @@ type StageStates struct { configs StageStatesCfg } type StageStatesCfg struct { - ctx context.Context bc core.BlockChain db kv.RwDB blockDBs []kv.RwDB @@ -34,7 +32,7 @@ func NewStageStates(cfg StageStatesCfg) *StageStates { } } -func NewStageStatesCfg(ctx context.Context, +func NewStageStatesCfg( bc core.BlockChain, db kv.RwDB, blockDBs []kv.RwDB, @@ -43,7 +41,6 @@ func NewStageStatesCfg(ctx context.Context, logProgress bool) StageStatesCfg { return StageStatesCfg{ - ctx: ctx, bc: bc, db: db, blockDBs: blockDBs, @@ -53,13 +50,8 @@ func NewStageStatesCfg(ctx context.Context, } } -func (stg *StageStates) SetStageContext(ctx context.Context) { - stg.configs.ctx = ctx -} - // Exec progresses States stage in the forward direction -func (stg *StageStates) Exec(firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) (err error) { - +func (stg *StageStates) Exec(ctx context.Context, firstCycle bool, invalidBlockRevert bool, s *StageState, reverter Reverter, tx kv.RwTx) (err error) { // for short range sync, skip this step if !s.state.initSync { return nil @@ -78,7 +70,7 @@ func (stg *StageStates) Exec(firstCycle bool, invalidBlockRevert bool, s *StageS useInternalTx := tx == nil if useInternalTx { var err error - tx, err = stg.configs.db.BeginRw(stg.configs.ctx) + tx, err = stg.configs.db.BeginRw(ctx) if err != nil { return err } @@ -94,7 +86,7 @@ func (stg *StageStates) Exec(firstCycle bool, invalidBlockRevert bool, s *StageS // prepare db transactions txs := make([]kv.RwTx, stg.configs.concurrency) for i := 0; i < stg.configs.concurrency; i++ { - txs[i], err = stg.configs.blockDBs[i].BeginRw(context.Background()) + txs[i], err = stg.configs.blockDBs[i].BeginRw(ctx) if err != nil { return err } @@ -219,19 +211,11 @@ func (stg *StageStates) Exec(firstCycle bool, invalidBlockRevert bool, s *StageS return nil } -func (stg *StageStates) insertChain(gbm *blockDownloadManager, - protocol syncProtocol, - lbls prometheus.Labels, - targetBN uint64) { - -} - -func (stg *StageStates) saveProgress(s *StageState, tx kv.RwTx) (err error) { - +func (stg *StageStates) saveProgress(ctx context.Context, s *StageState, tx kv.RwTx) (err error) { useInternalTx := tx == nil if useInternalTx { var err error - tx, err = stg.configs.db.BeginRw(context.Background()) + tx, err = stg.configs.db.BeginRw(ctx) if err != nil { return err } @@ -254,10 +238,10 @@ func (stg *StageStates) saveProgress(s *StageState, tx kv.RwTx) (err error) { return nil } -func (stg *StageStates) Revert(firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { +func (stg *StageStates) Revert(ctx context.Context, firstCycle bool, u *RevertState, s *StageState, tx kv.RwTx) (err error) { useInternalTx := tx == nil if useInternalTx { - tx, err = stg.configs.db.BeginRw(stg.configs.ctx) + tx, err = stg.configs.db.BeginRw(ctx) if err != nil { return err } @@ -276,10 +260,10 @@ func (stg *StageStates) Revert(firstCycle bool, u *RevertState, s *StageState, t return nil } -func (stg *StageStates) CleanUp(firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { +func (stg *StageStates) CleanUp(ctx context.Context, firstCycle bool, p *CleanUpState, tx kv.RwTx) (err error) { useInternalTx := tx == nil if useInternalTx { - tx, err = stg.configs.db.BeginRw(stg.configs.ctx) + tx, err = stg.configs.db.BeginRw(ctx) if err != nil { return err } diff --git a/api/service/stagedstreamsync/staged_stream_sync.go b/api/service/stagedstreamsync/staged_stream_sync.go index a03dd03b1a..6c1eec4c37 100644 --- a/api/service/stagedstreamsync/staged_stream_sync.go +++ b/api/service/stagedstreamsync/staged_stream_sync.go @@ -54,7 +54,6 @@ func (ib *InvalidBlock) addBadStream(bsID sttypes.StreamID) { } type StagedStreamSync struct { - ctx context.Context bc core.BlockChain isBeacon bool isExplorer bool @@ -101,7 +100,6 @@ type SyncCycle struct { } func (s *StagedStreamSync) Len() int { return len(s.stages) } -func (s *StagedStreamSync) Context() context.Context { return s.ctx } func (s *StagedStreamSync) Blockchain() core.BlockChain { return s.bc } func (s *StagedStreamSync) DB() kv.RwDB { return s.db } func (s *StagedStreamSync) IsBeacon() bool { return s.isBeacon } @@ -118,11 +116,11 @@ func (s *StagedStreamSync) NewRevertState(id SyncStageID, revertPoint uint64) *R return &RevertState{id, revertPoint, s} } -func (s *StagedStreamSync) CleanUpStageState(id SyncStageID, forwardProgress uint64, tx kv.Tx, db kv.RwDB) (*CleanUpState, error) { +func (s *StagedStreamSync) CleanUpStageState(ctx context.Context, id SyncStageID, forwardProgress uint64, tx kv.Tx, db kv.RwDB) (*CleanUpState, error) { var pruneProgress uint64 var err error - if errV := CreateView(context.Background(), db, tx, func(tx kv.Tx) error { + if errV := CreateView(ctx, db, tx, func(tx kv.Tx) error { pruneProgress, err = GetStageCleanUpProgress(tx, id, s.isBeacon) if err != nil { return err @@ -215,10 +213,10 @@ func (s *StagedStreamSync) SetCurrentStage(id SyncStageID) error { } // StageState retrieves the latest stage state from db -func (s *StagedStreamSync) StageState(stage SyncStageID, tx kv.Tx, db kv.RwDB) (*StageState, error) { +func (s *StagedStreamSync) StageState(ctx context.Context, stage SyncStageID, tx kv.Tx, db kv.RwDB) (*StageState, error) { var blockNum uint64 var err error - if errV := CreateView(context.Background(), db, tx, func(rtx kv.Tx) error { + if errV := CreateView(ctx, db, tx, func(rtx kv.Tx) error { blockNum, err = GetStageProgress(rtx, stage, s.isBeacon) if err != nil { return err @@ -232,7 +230,7 @@ func (s *StagedStreamSync) StageState(stage SyncStageID, tx kv.Tx, db kv.RwDB) ( } // cleanUp cleans up the stage by calling pruneStage -func (s *StagedStreamSync) cleanUp(fromStage int, db kv.RwDB, tx kv.RwTx, firstCycle bool) error { +func (s *StagedStreamSync) cleanUp(ctx context.Context, fromStage int, db kv.RwDB, tx kv.RwTx, firstCycle bool) error { found := false for i := 0; i < len(s.pruningOrder); i++ { if s.pruningOrder[i].ID == s.stages[fromStage].ID { @@ -241,7 +239,7 @@ func (s *StagedStreamSync) cleanUp(fromStage int, db kv.RwDB, tx kv.RwTx, firstC if !found || s.pruningOrder[i] == nil || s.pruningOrder[i].Disabled { continue } - if err := s.pruneStage(firstCycle, s.pruningOrder[i], db, tx); err != nil { + if err := s.pruneStage(ctx, firstCycle, s.pruningOrder[i], db, tx); err != nil { panic(err) } } @@ -249,7 +247,7 @@ func (s *StagedStreamSync) cleanUp(fromStage int, db kv.RwDB, tx kv.RwTx, firstC } // New creates a new StagedStreamSync instance -func New(ctx context.Context, +func New( bc core.BlockChain, db kv.RwDB, stagesList []*Stage, @@ -288,7 +286,6 @@ func New(ctx context.Context, status := newStatus() return &StagedStreamSync{ - ctx: ctx, bc: bc, isBeacon: isBeacon, db: db, @@ -309,8 +306,8 @@ func New(ctx context.Context, } // doGetCurrentNumberRequest returns estimated current block number and corresponding stream -func (s *StagedStreamSync) doGetCurrentNumberRequest() (uint64, sttypes.StreamID, error) { - ctx, cancel := context.WithTimeout(s.ctx, 10*time.Second) +func (s *StagedStreamSync) doGetCurrentNumberRequest(ctx context.Context) (uint64, sttypes.StreamID, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() bn, stid, err := s.protocol.GetCurrentBlockNumber(ctx, syncproto.WithHighPriority()) @@ -336,16 +333,8 @@ func (s *StagedStreamSync) checkHaveEnoughStreams() error { return nil } -// SetNewContext sets a new context for all stages -func (s *StagedStreamSync) SetNewContext(ctx context.Context) error { - for _, s := range s.stages { - s.Handler.SetStageContext(ctx) - } - return nil -} - // Run runs a full cycle of stages -func (s *StagedStreamSync) Run(db kv.RwDB, tx kv.RwTx, firstCycle bool) error { +func (s *StagedStreamSync) Run(ctx context.Context, db kv.RwDB, tx kv.RwTx, firstCycle bool) error { s.prevRevertPoint = nil s.timings = s.timings[:0] @@ -358,7 +347,7 @@ func (s *StagedStreamSync) Run(db kv.RwDB, tx kv.RwTx, firstCycle bool) error { if s.revertOrder[j] == nil || s.revertOrder[j].Disabled { continue } - if err := s.revertStage(firstCycle, s.revertOrder[j], db, tx); err != nil { + if err := s.revertStage(ctx, firstCycle, s.revertOrder[j], db, tx); err != nil { utils.Logger().Error(). Err(err). Interface("stage id", s.revertOrder[j].ID). @@ -383,7 +372,7 @@ func (s *StagedStreamSync) Run(db kv.RwDB, tx kv.RwTx, firstCycle bool) error { continue } - if err := s.runStage(stage, db, tx, firstCycle, s.invalidBlock.Active); err != nil { + if err := s.runStage(ctx, stage, db, tx, firstCycle, s.invalidBlock.Active); err != nil { utils.Logger().Error(). Err(err). Interface("stage id", stage.ID). @@ -393,7 +382,7 @@ func (s *StagedStreamSync) Run(db kv.RwDB, tx kv.RwTx, firstCycle bool) error { s.NextStage() } - if err := s.cleanUp(0, db, tx, firstCycle); err != nil { + if err := s.cleanUp(ctx, 0, db, tx, firstCycle); err != nil { utils.Logger().Error(). Err(err). Msgf(WrapStagedSyncMsg("stages cleanup failed")) @@ -414,7 +403,7 @@ func CreateView(ctx context.Context, db kv.RwDB, tx kv.Tx, f func(tx kv.Tx) erro if tx != nil { return f(tx) } - return db.View(context.Background(), func(etx kv.Tx) error { + return db.View(ctx, func(etx kv.Tx) error { return f(etx) }) } @@ -466,14 +455,14 @@ func printLogs(tx kv.RwTx, timings []Timing) error { } // runStage executes stage -func (s *StagedStreamSync) runStage(stage *Stage, db kv.RwDB, tx kv.RwTx, firstCycle bool, invalidBlockRevert bool) (err error) { +func (s *StagedStreamSync) runStage(ctx context.Context, stage *Stage, db kv.RwDB, tx kv.RwTx, firstCycle bool, invalidBlockRevert bool) (err error) { start := time.Now() - stageState, err := s.StageState(stage.ID, tx, db) + stageState, err := s.StageState(ctx, stage.ID, tx, db) if err != nil { return err } - if err = stage.Handler.Exec(firstCycle, invalidBlockRevert, stageState, s, tx); err != nil { + if err = stage.Handler.Exec(ctx, firstCycle, invalidBlockRevert, stageState, s, tx); err != nil { utils.Logger().Error(). Err(err). Interface("stage id", stage.ID). @@ -493,9 +482,9 @@ func (s *StagedStreamSync) runStage(stage *Stage, db kv.RwDB, tx kv.RwTx, firstC } // revertStage reverts stage -func (s *StagedStreamSync) revertStage(firstCycle bool, stage *Stage, db kv.RwDB, tx kv.RwTx) error { +func (s *StagedStreamSync) revertStage(ctx context.Context, firstCycle bool, stage *Stage, db kv.RwDB, tx kv.RwTx) error { start := time.Now() - stageState, err := s.StageState(stage.ID, tx, db) + stageState, err := s.StageState(ctx, stage.ID, tx, db) if err != nil { return err } @@ -510,7 +499,7 @@ func (s *StagedStreamSync) revertStage(firstCycle bool, stage *Stage, db kv.RwDB return err } - err = stage.Handler.Revert(firstCycle, revert, stageState, tx) + err = stage.Handler.Revert(ctx, firstCycle, revert, stageState, tx) if err != nil { return fmt.Errorf("[%s] %w", s.LogPrefix(), err) } @@ -526,15 +515,15 @@ func (s *StagedStreamSync) revertStage(firstCycle bool, stage *Stage, db kv.RwDB } // pruneStage cleans up the stage and logs the timing -func (s *StagedStreamSync) pruneStage(firstCycle bool, stage *Stage, db kv.RwDB, tx kv.RwTx) error { +func (s *StagedStreamSync) pruneStage(ctx context.Context, firstCycle bool, stage *Stage, db kv.RwDB, tx kv.RwTx) error { start := time.Now() - stageState, err := s.StageState(stage.ID, tx, db) + stageState, err := s.StageState(ctx, stage.ID, tx, db) if err != nil { return err } - prune, err := s.CleanUpStageState(stage.ID, stageState.BlockNumber, tx, db) + prune, err := s.CleanUpStageState(ctx, stage.ID, stageState.BlockNumber, tx, db) if err != nil { return err } @@ -542,7 +531,7 @@ func (s *StagedStreamSync) pruneStage(firstCycle bool, stage *Stage, db kv.RwDB, return err } - err = stage.Handler.CleanUp(firstCycle, prune, tx) + err = stage.Handler.CleanUp(ctx, firstCycle, prune, tx) if err != nil { return fmt.Errorf("[%s] %w", s.LogPrefix(), err) } diff --git a/api/service/stagedstreamsync/syncing.go b/api/service/stagedstreamsync/syncing.go index 75a15ff55a..ebef810b19 100644 --- a/api/service/stagedstreamsync/syncing.go +++ b/api/service/stagedstreamsync/syncing.go @@ -68,12 +68,12 @@ func CreateStagedSync(ctx context.Context, return nil, errInitDB } - stageHeadsCfg := NewStageHeadersCfg(ctx, bc, mainDB) - stageShortRangeCfg := NewStageShortRangeCfg(ctx, bc, mainDB) - stageSyncEpochCfg := NewStageEpochCfg(ctx, bc, mainDB) - stageBodiesCfg := NewStageBodiesCfg(ctx, bc, mainDB, dbs, config.Concurrency, protocol, isBeacon, logProgress) - stageStatesCfg := NewStageStatesCfg(ctx, bc, mainDB, dbs, config.Concurrency, logger, logProgress) - stageFinishCfg := NewStageFinishCfg(ctx, mainDB) + stageHeadsCfg := NewStageHeadersCfg(bc, mainDB) + stageShortRangeCfg := NewStageShortRangeCfg(bc, mainDB) + stageSyncEpochCfg := NewStageEpochCfg(bc, mainDB) + stageBodiesCfg := NewStageBodiesCfg(bc, mainDB, dbs, config.Concurrency, protocol, isBeacon, logProgress) + stageStatesCfg := NewStageStatesCfg(bc, mainDB, dbs, config.Concurrency, logger, logProgress) + stageFinishCfg := NewStageFinishCfg(mainDB) stages := DefaultStages(ctx, stageHeadsCfg, @@ -84,7 +84,7 @@ func CreateStagedSync(ctx context.Context, stageFinishCfg, ) - return New(ctx, + return New( bc, mainDB, stages, @@ -178,7 +178,7 @@ func (s *StagedStreamSync) doSync(downloaderContext context.Context, initSync bo var estimatedHeight uint64 if initSync { - if h, err := s.estimateCurrentNumber(); err != nil { + if h, err := s.estimateCurrentNumber(downloaderContext); err != nil { return 0, err } else { estimatedHeight = h @@ -197,8 +197,6 @@ func (s *StagedStreamSync) doSync(downloaderContext context.Context, initSync bo for { ctx, cancel := context.WithCancel(downloaderContext) - s.ctx = ctx - s.SetNewContext(ctx) n, err := s.doSyncCycle(ctx, initSync) if err != nil { @@ -234,7 +232,7 @@ func (s *StagedStreamSync) doSyncCycle(ctx context.Context, initSync bool) (int, var tx kv.RwTx if canRunCycleInOneTransaction { var err error - if tx, err = s.DB().BeginRw(context.Background()); err != nil { + if tx, err = s.DB().BeginRw(ctx); err != nil { return totalInserted, err } defer tx.Rollback() @@ -244,7 +242,7 @@ func (s *StagedStreamSync) doSyncCycle(ctx context.Context, initSync bool) (int, // Do one cycle of staged sync initialCycle := s.currentCycle.Number == 0 - if err := s.Run(s.DB(), tx, initialCycle); err != nil { + if err := s.Run(ctx, s.DB(), tx, initialCycle); err != nil { utils.Logger().Error(). Err(err). Bool("isBeacon", s.isBeacon). @@ -294,7 +292,7 @@ func (s *StagedStreamSync) checkPrerequisites() error { // estimateCurrentNumber roughly estimates the current block number. // The block number does not need to be exact, but just a temporary target of the iteration -func (s *StagedStreamSync) estimateCurrentNumber() (uint64, error) { +func (s *StagedStreamSync) estimateCurrentNumber(ctx context.Context) (uint64, error) { var ( cnResults = make(map[sttypes.StreamID]uint64) lock sync.Mutex @@ -304,7 +302,7 @@ func (s *StagedStreamSync) estimateCurrentNumber() (uint64, error) { for i := 0; i != s.config.Concurrency; i++ { go func() { defer wg.Done() - bn, stid, err := s.doGetCurrentNumberRequest() + bn, stid, err := s.doGetCurrentNumberRequest(ctx) if err != nil { s.logger.Err(err).Str("streamID", string(stid)). Msg(WrapStagedSyncMsg("getCurrentNumber request failed")) @@ -322,8 +320,8 @@ func (s *StagedStreamSync) estimateCurrentNumber() (uint64, error) { if len(cnResults) == 0 { select { - case <-s.ctx.Done(): - return 0, s.ctx.Err() + case <-ctx.Done(): + return 0, ctx.Err() default: } return 0, ErrZeroBlockResponse diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 2b80ecd8bc..af40c42bcb 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -682,7 +682,7 @@ func VerifyNewBlock(hooks *webhooks.Hooks, blockChain core.BlockChain, beaconCha Int("numTx", len(newBlock.Transactions())). Int("numStakingTx", len(newBlock.StakingTransactions())). Err(err). - Msg("[VerifyNewBlock] Cannot Verify New Block!!!") + Msgf("[VerifyNewBlock] Cannot Verify New Block!!!, blockHeight %d, myHeight %d", newBlock.NumberU64(), blockChain.CurrentHeader().NumberU64()) return errors.Errorf( "[VerifyNewBlock] Cannot Verify New Block!!! block-hash %s txn-count %d", newBlock.Hash().Hex(), From d1f986ffa567fcc6f3a278426a4412b247a3abe6 Mon Sep 17 00:00:00 2001 From: Max <82761650+MaxMustermann2@users.noreply.github.com> Date: Wed, 21 Jun 2023 16:03:26 +0000 Subject: [PATCH 418/420] build: update base distribution in Travis (#4451) On Docker versions < 20.10.9, `apt update` fails due to the use of syscall `clone3` by `Glibc >= 2.34`. This change upgrades the base distribution used by Travis to `jammy`, which contains Docker engine 20.10.12. See https://docs.travis-ci.com/user/reference/jammy/#docker and moby/moby#42681 for reference. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cf99b47e61..dd92b53df5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ os: linux -dist: bionic +dist: jammy language: go go: - 1.19.5 From 21b4988668ea57fab526759ca59772494e3a4fe3 Mon Sep 17 00:00:00 2001 From: Soph <35721420+sophoah@users.noreply.github.com> Date: Thu, 22 Jun 2023 22:56:22 +0700 Subject: [PATCH 419/420] [github action] build and push harmony proto docker image (#4453) --- .../workflows/build-docker-proto-image.yaml | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/build-docker-proto-image.yaml diff --git a/.github/workflows/build-docker-proto-image.yaml b/.github/workflows/build-docker-proto-image.yaml new file mode 100644 index 0000000000..3edab75795 --- /dev/null +++ b/.github/workflows/build-docker-proto-image.yaml @@ -0,0 +1,35 @@ +name: Build and Push harmony proto Docker Image + +on: + workflow_dispatch: + +jobs: + build_and_push: + runs-on: ubuntu-latest + + steps: + - name: Checkout harmony core code + uses: actions/checkout@v3 + with: + path: harmony + ref: ${{ github.ref }} + fetch-depth: 0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push + uses: docker/build-push-action@v3 + with: + context: ./harmony/api/service/legacysync/downloader + file: ./harmony/api/service/legacysync/downloader/Proto.Dockerfile + push: true + platforms: linux/amd64,linux/arm64 + tags: | + harmonyone/harmony-proto:latest \ No newline at end of file From 32e591358e657a80ef2e4545b81b83ef2016c8d5 Mon Sep 17 00:00:00 2001 From: Max <82761650+MaxMustermann2@users.noreply.github.com> Date: Fri, 23 Jun 2023 00:19:54 +0000 Subject: [PATCH 420/420] rpc: add `net_listening` function (#4446) --- rpc/net.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rpc/net.go b/rpc/net.go index 45c122e4bd..70c39fe461 100644 --- a/rpc/net.go +++ b/rpc/net.go @@ -69,3 +69,8 @@ func (s *PublicNetService) Version(ctx context.Context) interface{} { return fmt.Sprintf("%d", s.chainID) } } + +// Listening returns an indication if the node is listening for network connections. +func (s *PublicNetService) Listening() bool { + return true // always listening +}