From 35e40b71da5b144d3eecf51dc5b7e07fa12299de Mon Sep 17 00:00:00 2001 From: envestcc Date: Tue, 25 Jul 2023 19:57:23 +0800 Subject: [PATCH 01/13] Add error return to StartHeight method --- blockchain/blockdao/blockindexer.go | 7 +++++-- blockindex/contractstaking/indexer.go | 4 ++-- blockindex/contractstaking/indexer_test.go | 4 +++- blockindex/sgd_indexer.go | 4 ++-- blockindex/sgd_indexer_test.go | 7 +++++-- test/mock/mock_blockdao/mock_blockindexer_withstart.go | 5 +++-- 6 files changed, 20 insertions(+), 11 deletions(-) diff --git a/blockchain/blockdao/blockindexer.go b/blockchain/blockdao/blockindexer.go index dfd70241ae..8c0ca6a4b3 100644 --- a/blockchain/blockdao/blockindexer.go +++ b/blockchain/blockdao/blockindexer.go @@ -32,7 +32,7 @@ type ( BlockIndexerWithStart interface { BlockIndexer // StartHeight returns the start height of the indexer - StartHeight() uint64 + StartHeight() (uint64, error) } // BlockIndexerChecker defines a checker of block indexer @@ -76,7 +76,10 @@ func (bic *BlockIndexerChecker) CheckIndexer(ctx context.Context, indexer BlockI } startHeight := tipHeight + 1 if indexerWS, ok := indexer.(BlockIndexerWithStart); ok { - indexStartHeight := indexerWS.StartHeight() + indexStartHeight, err := indexerWS.StartHeight() + if err != nil { + return err + } if indexStartHeight > startHeight { startHeight = indexStartHeight } diff --git a/blockindex/contractstaking/indexer.go b/blockindex/contractstaking/indexer.go index 5adcf8a386..516217a1c3 100644 --- a/blockindex/contractstaking/indexer.go +++ b/blockindex/contractstaking/indexer.go @@ -75,8 +75,8 @@ func (s *Indexer) Height() (uint64, error) { } // StartHeight returns the start height of the indexer -func (s *Indexer) StartHeight() uint64 { - return s.contractDeployHeight +func (s *Indexer) StartHeight() (uint64, error) { + return s.contractDeployHeight, nil } // CandidateVotes returns the candidate votes diff --git a/blockindex/contractstaking/indexer_test.go b/blockindex/contractstaking/indexer_test.go index 20a61cc885..efa1ab17ca 100644 --- a/blockindex/contractstaking/indexer_test.go +++ b/blockindex/contractstaking/indexer_test.go @@ -101,7 +101,9 @@ func TestContractStakingIndexerLoadCache(t *testing.T) { newHeight, err := newIndexer.Height() r.NoError(err) r.Equal(height, newHeight) - r.Equal(startHeight, newIndexer.StartHeight()) + newStartHeight, err := newIndexer.StartHeight() + r.NoError(err) + r.Equal(startHeight, newStartHeight) r.EqualValues(1, newIndexer.TotalBucketCount()) r.NoError(newIndexer.Stop(context.Background())) } diff --git a/blockindex/sgd_indexer.go b/blockindex/sgd_indexer.go index 22cd472b78..16bc0c0191 100644 --- a/blockindex/sgd_indexer.go +++ b/blockindex/sgd_indexer.go @@ -279,8 +279,8 @@ func (sgd *sgdRegistry) Height() (uint64, error) { } // StartHeight returns the start height of the indexer -func (sgd *sgdRegistry) StartHeight() uint64 { - return sgd.startHeight +func (sgd *sgdRegistry) StartHeight() (uint64, error) { + return sgd.startHeight, nil } // PutBlock puts a block into SGDIndexer diff --git a/blockindex/sgd_indexer_test.go b/blockindex/sgd_indexer_test.go index 7097f53588..db24a612ff 100644 --- a/blockindex/sgd_indexer_test.go +++ b/blockindex/sgd_indexer_test.go @@ -9,6 +9,8 @@ import ( "github.com/iotexproject/go-pkgs/hash" "github.com/iotexproject/iotex-address/address" + "github.com/stretchr/testify/require" + "github.com/iotexproject/iotex-core/action" "github.com/iotexproject/iotex-core/blockchain/block" "github.com/iotexproject/iotex-core/blockchain/genesis" @@ -16,7 +18,6 @@ import ( "github.com/iotexproject/iotex-core/state" "github.com/iotexproject/iotex-core/test/identityset" "github.com/iotexproject/iotex-core/testutil" - "github.com/stretchr/testify/require" ) const ( @@ -54,7 +55,9 @@ func TestNewSGDRegistry(t *testing.T) { }() nonce := uint64(0) - r.Equal(nonce, sgdRegistry.StartHeight()) + startHeight, err := sgdRegistry.StartHeight() + r.NoError(err) + r.Equal(nonce, startHeight) hh, err := sgdRegistry.Height() r.NoError(err) r.Equal(nonce, hh) diff --git a/test/mock/mock_blockdao/mock_blockindexer_withstart.go b/test/mock/mock_blockdao/mock_blockindexer_withstart.go index 2f8a029269..e0d16c0eaf 100644 --- a/test/mock/mock_blockdao/mock_blockindexer_withstart.go +++ b/test/mock/mock_blockdao/mock_blockindexer_withstart.go @@ -93,11 +93,12 @@ func (mr *MockBlockIndexerWithStartMockRecorder) Start(arg0 interface{}) *gomock } // StartHeight mocks base method. -func (m *MockBlockIndexerWithStart) StartHeight() uint64 { +func (m *MockBlockIndexerWithStart) StartHeight() (uint64, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "StartHeight") ret0, _ := ret[0].(uint64) - return ret0 + ret1, _ := ret[1].(error) + return ret0, ret1 } // StartHeight indicates an expected call of StartHeight. From f7502e85271db1d8216695d104c69ccc348f2963 Mon Sep 17 00:00:00 2001 From: envestcc Date: Tue, 25 Jul 2023 19:59:20 +0800 Subject: [PATCH 02/13] implement IndexerGroup --- blockindex/indexer_group.go | 123 ++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 blockindex/indexer_group.go diff --git a/blockindex/indexer_group.go b/blockindex/indexer_group.go new file mode 100644 index 0000000000..3bfeadfeb3 --- /dev/null +++ b/blockindex/indexer_group.go @@ -0,0 +1,123 @@ +// Copyright (c) 2023 IoTeX Foundation +// This source code is provided 'as is' and no warranties are given as to title or non-infringement, merchantability +// or fitness for purpose and, to the extent permitted by law, all liability for your use of the code is disclaimed. +// This source code is governed by Apache License 2.0 that can be found in the LICENSE file. + +package blockindex + +import ( + "context" + + "github.com/iotexproject/iotex-core/blockchain/block" + "github.com/iotexproject/iotex-core/blockchain/blockdao" +) + +// Index group is a special index that includes multiple indexes, +// which stay in sync when blocks are added. +type IndexerGroup struct { + indexers []blockdao.BlockIndexer +} + +// NewIndexerGroup creates a new indexer group +func NewIndexerGroup(indexers ...blockdao.BlockIndexer) *IndexerGroup { + return &IndexerGroup{indexers: indexers} +} + +// Start starts the indexer group +func (ig *IndexerGroup) Start(ctx context.Context) error { + for _, indexer := range ig.indexers { + if err := indexer.Start(ctx); err != nil { + return err + } + } + return nil +} + +// Stop stops the indexer group +func (ig *IndexerGroup) Stop(ctx context.Context) error { + for _, indexer := range ig.indexers { + if err := indexer.Stop(ctx); err != nil { + return err + } + } + return nil +} + +// PutBlock puts a block into the indexers in the group +func (ig *IndexerGroup) PutBlock(ctx context.Context, blk *block.Block) error { + for _, indexer := range ig.indexers { + // check if the indexer is a BlockIndexerWithStart + if indexerWithStart, ok := indexer.(blockdao.BlockIndexerWithStart); ok { + startHeight, err := indexerWithStart.StartHeight() + if err != nil { + return err + } + if blk.Height() < startHeight { + continue + } + } + // check if the block is higher than the indexer's height + height, err := indexer.Height() + if err != nil { + return err + } + if blk.Height() <= height { + continue + } + // put block + if err := indexer.PutBlock(ctx, blk); err != nil { + return err + } + } + return nil +} + +// DeleteTipBlock deletes the tip block from the indexers in the group +func (ig *IndexerGroup) DeleteTipBlock(ctx context.Context, blk *block.Block) error { + for _, indexer := range ig.indexers { + if err := indexer.DeleteTipBlock(ctx, blk); err != nil { + return err + } + } + return nil +} + +// StartHeight returns the minimum start height of the indexers in the group +func (ig *IndexerGroup) StartHeight() (uint64, error) { + var result uint64 + for i, indexer := range ig.indexers { + tipHeight, err := indexer.Height() + if err != nil { + return 0, err + } + indexStartHeight := tipHeight + 1 + if indexerWithStart, ok := indexer.(blockdao.BlockIndexerWithStart); ok { + startHeight, err := indexerWithStart.StartHeight() + if err != nil { + return 0, err + } + if startHeight > indexStartHeight { + indexStartHeight = startHeight + } + } + if i == 0 || indexStartHeight < result { + result = indexStartHeight + } + } + return result, nil +} + +// Height returns the minimum height of the indexers in the group +func (ig *IndexerGroup) Height() (uint64, error) { + var height uint64 + for _, indexer := range ig.indexers { + h, err := indexer.Height() + if err != nil { + return 0, err + } + if height == 0 || h < height { + height = h + } + } + return height, nil +} From e4d9dbf9173883a069c73595cb0f43985310792a Mon Sep 17 00:00:00 2001 From: envestcc Date: Wed, 26 Jul 2023 08:46:28 +0800 Subject: [PATCH 03/13] move factory and contractstaking indexer into a group --- chainservice/builder.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/chainservice/builder.go b/chainservice/builder.go index b7bac80dbb..682f26d7dc 100644 --- a/chainservice/builder.go +++ b/chainservice/builder.go @@ -257,7 +257,11 @@ func (builder *Builder) buildBlockDAO(forTest bool) error { } var indexers []blockdao.BlockIndexer - indexers = append(indexers, builder.cs.factory) + if builder.cs.contractStakingIndexer != nil { + indexers = append(indexers, blockindex.NewIndexerGroup(builder.cs.factory, builder.cs.contractStakingIndexer)) + } else { + indexers = append(indexers, builder.cs.factory) + } if !builder.cfg.Chain.EnableAsyncIndexWrite && builder.cs.indexer != nil { indexers = append(indexers, builder.cs.indexer) } @@ -267,9 +271,6 @@ func (builder *Builder) buildBlockDAO(forTest bool) error { if builder.cs.sgdIndexer != nil { indexers = append(indexers, builder.cs.sgdIndexer) } - if builder.cs.contractStakingIndexer != nil { - indexers = append(indexers, builder.cs.contractStakingIndexer) - } if forTest { builder.cs.blockdao = blockdao.NewBlockDAOInMemForTest(indexers) } else { From 0e32daeb8cdf6e7f8b10b267b8293f4e4cd3330e Mon Sep 17 00:00:00 2001 From: envestcc Date: Wed, 26 Jul 2023 08:47:08 +0800 Subject: [PATCH 04/13] add unit test --- blockchain/blockdao/blockindexer_test.go | 2 +- blockindex/indexer_group.go | 2 +- blockindex/indexer_group_test.go | 68 ++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 blockindex/indexer_group_test.go diff --git a/blockchain/blockdao/blockindexer_test.go b/blockchain/blockdao/blockindexer_test.go index d80ae9788e..26a11d1241 100644 --- a/blockchain/blockdao/blockindexer_test.go +++ b/blockchain/blockdao/blockindexer_test.go @@ -125,7 +125,7 @@ func TestCheckIndexerWithStart(t *testing.T) { }).AnyTimes() mockDao.EXPECT().GetReceipts(gomock.Any()).Return(nil, nil).AnyTimes() indexer.EXPECT().Height().Return(c.indexerTipHeight, nil).Times(1) - indexer.EXPECT().StartHeight().Return(c.indexerStartHeight).AnyTimes() + indexer.EXPECT().StartHeight().Return(c.indexerStartHeight, nil).AnyTimes() indexer.EXPECT().PutBlock(gomock.Any(), gomock.Any()).DoAndReturn(func(arg0 context.Context, arg1 *block.Block) error { putBlocks = append(putBlocks, arg1) return nil diff --git a/blockindex/indexer_group.go b/blockindex/indexer_group.go index 3bfeadfeb3..86ea77e8c8 100644 --- a/blockindex/indexer_group.go +++ b/blockindex/indexer_group.go @@ -12,7 +12,7 @@ import ( "github.com/iotexproject/iotex-core/blockchain/blockdao" ) -// Index group is a special index that includes multiple indexes, +// IndexerGroup is a special index that includes multiple indexes, // which stay in sync when blocks are added. type IndexerGroup struct { indexers []blockdao.BlockIndexer diff --git a/blockindex/indexer_group_test.go b/blockindex/indexer_group_test.go new file mode 100644 index 0000000000..d3a3a4e95b --- /dev/null +++ b/blockindex/indexer_group_test.go @@ -0,0 +1,68 @@ +// Copyright (c) 2023 IoTeX Foundation +// This source code is provided 'as is' and no warranties are given as to title or non-infringement, merchantability +// or fitness for purpose and, to the extent permitted by law, all liability for your use of the code is disclaimed. +// This source code is governed by Apache License 2.0 that can be found in the LICENSE file. + +package blockindex + +import ( + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + "github.com/iotexproject/iotex-core/blockchain/blockdao" + "github.com/iotexproject/iotex-core/test/mock/mock_blockdao" +) + +func TestIndexerGroup_StartHeight(t *testing.T) { + require := require.New(t) + ctrl := gomock.NewController(t) + + cases := []struct { + name string + indexers [][2]uint64 // [startHeight, height] + expect uint64 + }{ + {"no indexers", nil, 0}, + {"one indexer without start height", [][2]uint64{{0, 100}}, 101}, + {"one indexer with start height I", [][2]uint64{{100, 200}}, 201}, + {"one indexer with start height II", [][2]uint64{{300, 200}}, 300}, + {"two indexers with start height I", [][2]uint64{{100, 200}, {200, 300}}, 201}, + {"two indexers with start height II", [][2]uint64{{100, 200}, {400, 300}}, 201}, + {"two indexers with start height III", [][2]uint64{{100, 350}, {400, 300}}, 351}, + {"two indexers one with start height I", [][2]uint64{{0, 1}, {150, 1}}, 2}, + {"two indexers one with start height II", [][2]uint64{{0, 1}, {150, 200}}, 2}, + {"two indexers one with start height III", [][2]uint64{{0, 200}, {250, 1}}, 201}, + {"two indexers one with start height IV", [][2]uint64{{0, 200}, {150, 1}}, 150}, + {"two indexers I", [][2]uint64{{0, 5}, {0, 1}}, 2}, + {"two indexers II", [][2]uint64{{0, 5}, {0, 5}}, 6}, + {"two indexers III", [][2]uint64{{0, 5}, {0, 6}}, 6}, + {"multiple indexers I", [][2]uint64{{0, 5}, {0, 6}, {0, 7}}, 6}, + {"multiple indexers II", [][2]uint64{{0, 5}, {10, 6}, {0, 7}}, 6}, + {"multiple indexers III", [][2]uint64{{10, 5}, {0, 6}, {0, 7}}, 7}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + var indexers []blockdao.BlockIndexer + for _, indexer := range c.indexers { + if indexer[0] > 0 { + mockIndexerWithStart := mock_blockdao.NewMockBlockIndexerWithStart(ctrl) + mockIndexerWithStart.EXPECT().StartHeight().Return(indexer[0]).Times(1) + mockIndexerWithStart.EXPECT().Height().Return(indexer[1], nil).Times(1) + indexers = append(indexers, mockIndexerWithStart) + } else { + mockIndexer := mock_blockdao.NewMockBlockIndexer(ctrl) + mockIndexer.EXPECT().Height().Return(indexer[1], nil).Times(1) + indexers = append(indexers, mockIndexer) + } + } + ig := NewIndexerGroup(indexers...) + height, err := ig.StartHeight() + require.NoError(err) + require.Equal(c.expect, height) + }) + } + +} From 40c3b7106b82f956fca1ca665b52c70db776606a Mon Sep 17 00:00:00 2001 From: envestcc Date: Thu, 27 Jul 2023 21:48:00 +0800 Subject: [PATCH 05/13] introduce dependentIndexers --- blockindex/dependent_indexers.go | 80 +++++++++++++++++++++++++++ blockindex/dependent_indexers_test.go | 47 ++++++++++++++++ go.mod | 1 + go.sum | 1 + 4 files changed, 129 insertions(+) create mode 100644 blockindex/dependent_indexers.go create mode 100644 blockindex/dependent_indexers_test.go diff --git a/blockindex/dependent_indexers.go b/blockindex/dependent_indexers.go new file mode 100644 index 0000000000..f15fe1717a --- /dev/null +++ b/blockindex/dependent_indexers.go @@ -0,0 +1,80 @@ +// Copyright (c) 2023 IoTeX Foundation +// This source code is provided 'as is' and no warranties are given as to title or non-infringement, merchantability +// or fitness for purpose and, to the extent permitted by law, all liability for your use of the code is disclaimed. +// This source code is governed by Apache License 2.0 that can be found in the LICENSE file. + +package blockindex + +import ( + "gonum.org/v1/gonum/graph/simple" + "gonum.org/v1/gonum/graph/topo" + + "github.com/iotexproject/iotex-core/blockchain/blockdao" +) + +type ( + // DependentIndexers is a struct that represents a group of indexers with dependencies between them. + DependentIndexers struct { + *IndexerGroup + } + + // graphNodeWrapper is a wrapper of blockdao.BlockIndexer that implements the gonum graph.Node interface + graphNodeWrapper struct { + indexer blockdao.BlockIndexer + id int64 + } + + // graphNodeWrapperBuilder is a builder of graphNodeWrapper + graphNodeWrapperBuilder struct { + wrapperMap map[blockdao.BlockIndexer]*graphNodeWrapper + count int64 + } +) + +// NewDependentIndexers creates a new dependent indexers. +// the dependencies are specified as pairs of indexers, where the first indexer depends on the second indexer. +// it uses a topological sort to determine the order in which the indexers should be added to the group +func NewDependentIndexers(dependencies ...[2]blockdao.BlockIndexer) (*DependentIndexers, error) { + // build the graph + g := simple.NewDirectedGraph() + builder := &graphNodeWrapperBuilder{ + wrapperMap: make(map[blockdao.BlockIndexer]*graphNodeWrapper), + } + for _, dependency := range dependencies { + g.SetEdge(g.NewEdge(builder.Build(dependency[0]), builder.Build(dependency[1]))) + } + // topological sort + order, err := topo.SortStabilized(g, nil) + if err != nil { + return nil, err + } + // build the indexers + var indexers []blockdao.BlockIndexer + for _, node := range order { + indexers = append(indexers, node.(*graphNodeWrapper).indexer) + } + return &DependentIndexers{IndexerGroup: NewIndexerGroup(indexers...)}, nil +} + +// ID returns the id of the wrapper +func (w *graphNodeWrapper) ID() int64 { + return w.id +} + +// Build builds a graphNodeWrapper +func (b *graphNodeWrapperBuilder) Build(indexer blockdao.BlockIndexer) *graphNodeWrapper { + // return if the wrapper is already built + if wrapper, ok := b.wrapperMap[indexer]; ok { + return wrapper + } + + // build the wrapper + id := b.count + b.count++ + wrapper := &graphNodeWrapper{ + indexer: indexer, + id: id, + } + b.wrapperMap[indexer] = wrapper + return wrapper +} diff --git a/blockindex/dependent_indexers_test.go b/blockindex/dependent_indexers_test.go new file mode 100644 index 0000000000..4bbdc6989a --- /dev/null +++ b/blockindex/dependent_indexers_test.go @@ -0,0 +1,47 @@ +// Copyright (c) 2023 IoTeX Foundation +// This source code is provided 'as is' and no warranties are given as to title or non-infringement, merchantability +// or fitness for purpose and, to the extent permitted by law, all liability for your use of the code is disclaimed. +// This source code is governed by Apache License 2.0 that can be found in the LICENSE file. + +package blockindex + +import ( + "strconv" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + "github.com/iotexproject/iotex-core/blockchain/blockdao" + "github.com/iotexproject/iotex-core/test/mock/mock_blockdao" +) + +func TestDependentIndexers(t *testing.T) { + require := require.New(t) + ctrl := gomock.NewController(t) + + index1 := mock_blockdao.NewMockBlockIndexer(ctrl) + index2 := mock_blockdao.NewMockBlockIndexer(ctrl) + index3 := mock_blockdao.NewMockBlockIndexer(ctrl) + + cases := []struct { + dependencies [][2]blockdao.BlockIndexer + expect []blockdao.BlockIndexer + }{ + {[][2]blockdao.BlockIndexer{{index1, index2}, {index1, index3}}, []blockdao.BlockIndexer{index1, index2, index3}}, + {[][2]blockdao.BlockIndexer{{index1, index2}, {index2, index3}}, []blockdao.BlockIndexer{index1, index2, index3}}, + {[][2]blockdao.BlockIndexer{{index1, index2}, {index3, index2}}, []blockdao.BlockIndexer{index1, index3, index2}}, + } + + for i, c := range cases { + name := strconv.FormatInt(int64(i), 10) + t.Run(name, func(t *testing.T) { + dependentIndexers, err := NewDependentIndexers(c.dependencies...) + require.NoError(err) + require.Equal(len(c.expect), len(dependentIndexers.indexers)) + for i := range c.expect { + require.True(c.expect[i] == dependentIndexers.indexers[i]) + } + }) + } +} diff --git a/go.mod b/go.mod index 0447c22750..8b387cbb89 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,7 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.34.0 golang.org/x/exp v0.0.0-20230206171751-46f607a40771 golang.org/x/text v0.7.0 + gonum.org/v1/gonum v0.6.0 ) require ( diff --git a/go.sum b/go.sum index a065dab82b..5e7c83e631 100644 --- a/go.sum +++ b/go.sum @@ -1624,6 +1624,7 @@ golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= golang.org/x/xerrors v0.0.0-20190212162355-a5947ffaace3/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.6.0 h1:DJy6UzXbahnGUf1ujUNkh/NEtK14qMo2nvlBPs4U5yw= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= From 9f4f8587b509f2ac00a6ae321e28d403ec381d09 Mon Sep 17 00:00:00 2001 From: envestcc Date: Thu, 27 Jul 2023 21:49:27 +0800 Subject: [PATCH 06/13] use dependentIndexers --- blockindex/indexer_group.go | 4 ++-- blockindex/indexer_group_test.go | 2 +- chainservice/builder.go | 16 ++++++++++++---- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/blockindex/indexer_group.go b/blockindex/indexer_group.go index 86ea77e8c8..60beb91a81 100644 --- a/blockindex/indexer_group.go +++ b/blockindex/indexer_group.go @@ -110,12 +110,12 @@ func (ig *IndexerGroup) StartHeight() (uint64, error) { // Height returns the minimum height of the indexers in the group func (ig *IndexerGroup) Height() (uint64, error) { var height uint64 - for _, indexer := range ig.indexers { + for i, indexer := range ig.indexers { h, err := indexer.Height() if err != nil { return 0, err } - if height == 0 || h < height { + if i == 0 || h < height { height = h } } diff --git a/blockindex/indexer_group_test.go b/blockindex/indexer_group_test.go index d3a3a4e95b..8321130f29 100644 --- a/blockindex/indexer_group_test.go +++ b/blockindex/indexer_group_test.go @@ -49,7 +49,7 @@ func TestIndexerGroup_StartHeight(t *testing.T) { for _, indexer := range c.indexers { if indexer[0] > 0 { mockIndexerWithStart := mock_blockdao.NewMockBlockIndexerWithStart(ctrl) - mockIndexerWithStart.EXPECT().StartHeight().Return(indexer[0]).Times(1) + mockIndexerWithStart.EXPECT().StartHeight().Return(indexer[0], nil).Times(1) mockIndexerWithStart.EXPECT().Height().Return(indexer[1], nil).Times(1) indexers = append(indexers, mockIndexerWithStart) } else { diff --git a/chainservice/builder.go b/chainservice/builder.go index 682f26d7dc..cc55993e8a 100644 --- a/chainservice/builder.go +++ b/chainservice/builder.go @@ -257,8 +257,19 @@ func (builder *Builder) buildBlockDAO(forTest bool) error { } var indexers []blockdao.BlockIndexer + dependencies := [][2]blockdao.BlockIndexer{} if builder.cs.contractStakingIndexer != nil { - indexers = append(indexers, blockindex.NewIndexerGroup(builder.cs.factory, builder.cs.contractStakingIndexer)) + dependencies = append(dependencies, [2]blockdao.BlockIndexer{builder.cs.factory, builder.cs.contractStakingIndexer}) + } + if builder.cs.sgdIndexer != nil { + dependencies = append(dependencies, [2]blockdao.BlockIndexer{builder.cs.factory, builder.cs.sgdIndexer}) + } + if len(dependencies) > 0 { + dependentIndexer, err := blockindex.NewDependentIndexers(dependencies...) + if err != nil { + return errors.Wrapf(err, "failed to create dependent indexers") + } + indexers = append(indexers, dependentIndexer) } else { indexers = append(indexers, builder.cs.factory) } @@ -268,9 +279,6 @@ func (builder *Builder) buildBlockDAO(forTest bool) error { if builder.cs.bfIndexer != nil { indexers = append(indexers, builder.cs.bfIndexer) } - if builder.cs.sgdIndexer != nil { - indexers = append(indexers, builder.cs.sgdIndexer) - } if forTest { builder.cs.blockdao = blockdao.NewBlockDAOInMemForTest(indexers) } else { From ffc80ea0ba6fa02edcdd77f2894180142246906f Mon Sep 17 00:00:00 2001 From: envestcc Date: Fri, 28 Jul 2023 10:36:37 +0800 Subject: [PATCH 07/13] remove over-designed dependentIndexers --- blockindex/dependent_indexers.go | 80 --------------------------- blockindex/dependent_indexers_test.go | 47 ---------------- chainservice/builder.go | 15 ++--- 3 files changed, 6 insertions(+), 136 deletions(-) delete mode 100644 blockindex/dependent_indexers.go delete mode 100644 blockindex/dependent_indexers_test.go diff --git a/blockindex/dependent_indexers.go b/blockindex/dependent_indexers.go deleted file mode 100644 index f15fe1717a..0000000000 --- a/blockindex/dependent_indexers.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) 2023 IoTeX Foundation -// This source code is provided 'as is' and no warranties are given as to title or non-infringement, merchantability -// or fitness for purpose and, to the extent permitted by law, all liability for your use of the code is disclaimed. -// This source code is governed by Apache License 2.0 that can be found in the LICENSE file. - -package blockindex - -import ( - "gonum.org/v1/gonum/graph/simple" - "gonum.org/v1/gonum/graph/topo" - - "github.com/iotexproject/iotex-core/blockchain/blockdao" -) - -type ( - // DependentIndexers is a struct that represents a group of indexers with dependencies between them. - DependentIndexers struct { - *IndexerGroup - } - - // graphNodeWrapper is a wrapper of blockdao.BlockIndexer that implements the gonum graph.Node interface - graphNodeWrapper struct { - indexer blockdao.BlockIndexer - id int64 - } - - // graphNodeWrapperBuilder is a builder of graphNodeWrapper - graphNodeWrapperBuilder struct { - wrapperMap map[blockdao.BlockIndexer]*graphNodeWrapper - count int64 - } -) - -// NewDependentIndexers creates a new dependent indexers. -// the dependencies are specified as pairs of indexers, where the first indexer depends on the second indexer. -// it uses a topological sort to determine the order in which the indexers should be added to the group -func NewDependentIndexers(dependencies ...[2]blockdao.BlockIndexer) (*DependentIndexers, error) { - // build the graph - g := simple.NewDirectedGraph() - builder := &graphNodeWrapperBuilder{ - wrapperMap: make(map[blockdao.BlockIndexer]*graphNodeWrapper), - } - for _, dependency := range dependencies { - g.SetEdge(g.NewEdge(builder.Build(dependency[0]), builder.Build(dependency[1]))) - } - // topological sort - order, err := topo.SortStabilized(g, nil) - if err != nil { - return nil, err - } - // build the indexers - var indexers []blockdao.BlockIndexer - for _, node := range order { - indexers = append(indexers, node.(*graphNodeWrapper).indexer) - } - return &DependentIndexers{IndexerGroup: NewIndexerGroup(indexers...)}, nil -} - -// ID returns the id of the wrapper -func (w *graphNodeWrapper) ID() int64 { - return w.id -} - -// Build builds a graphNodeWrapper -func (b *graphNodeWrapperBuilder) Build(indexer blockdao.BlockIndexer) *graphNodeWrapper { - // return if the wrapper is already built - if wrapper, ok := b.wrapperMap[indexer]; ok { - return wrapper - } - - // build the wrapper - id := b.count - b.count++ - wrapper := &graphNodeWrapper{ - indexer: indexer, - id: id, - } - b.wrapperMap[indexer] = wrapper - return wrapper -} diff --git a/blockindex/dependent_indexers_test.go b/blockindex/dependent_indexers_test.go deleted file mode 100644 index 4bbdc6989a..0000000000 --- a/blockindex/dependent_indexers_test.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2023 IoTeX Foundation -// This source code is provided 'as is' and no warranties are given as to title or non-infringement, merchantability -// or fitness for purpose and, to the extent permitted by law, all liability for your use of the code is disclaimed. -// This source code is governed by Apache License 2.0 that can be found in the LICENSE file. - -package blockindex - -import ( - "strconv" - "testing" - - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/require" - - "github.com/iotexproject/iotex-core/blockchain/blockdao" - "github.com/iotexproject/iotex-core/test/mock/mock_blockdao" -) - -func TestDependentIndexers(t *testing.T) { - require := require.New(t) - ctrl := gomock.NewController(t) - - index1 := mock_blockdao.NewMockBlockIndexer(ctrl) - index2 := mock_blockdao.NewMockBlockIndexer(ctrl) - index3 := mock_blockdao.NewMockBlockIndexer(ctrl) - - cases := []struct { - dependencies [][2]blockdao.BlockIndexer - expect []blockdao.BlockIndexer - }{ - {[][2]blockdao.BlockIndexer{{index1, index2}, {index1, index3}}, []blockdao.BlockIndexer{index1, index2, index3}}, - {[][2]blockdao.BlockIndexer{{index1, index2}, {index2, index3}}, []blockdao.BlockIndexer{index1, index2, index3}}, - {[][2]blockdao.BlockIndexer{{index1, index2}, {index3, index2}}, []blockdao.BlockIndexer{index1, index3, index2}}, - } - - for i, c := range cases { - name := strconv.FormatInt(int64(i), 10) - t.Run(name, func(t *testing.T) { - dependentIndexers, err := NewDependentIndexers(c.dependencies...) - require.NoError(err) - require.Equal(len(c.expect), len(dependentIndexers.indexers)) - for i := range c.expect { - require.True(c.expect[i] == dependentIndexers.indexers[i]) - } - }) - } -} diff --git a/chainservice/builder.go b/chainservice/builder.go index cc55993e8a..c64b611b94 100644 --- a/chainservice/builder.go +++ b/chainservice/builder.go @@ -257,19 +257,16 @@ func (builder *Builder) buildBlockDAO(forTest bool) error { } var indexers []blockdao.BlockIndexer - dependencies := [][2]blockdao.BlockIndexer{} + // indexers in synchronizedIndexers will be putblock one by one for every block + synchronizedIndexers := []blockdao.BlockIndexer{builder.cs.factory} if builder.cs.contractStakingIndexer != nil { - dependencies = append(dependencies, [2]blockdao.BlockIndexer{builder.cs.factory, builder.cs.contractStakingIndexer}) + synchronizedIndexers = append(synchronizedIndexers, builder.cs.contractStakingIndexer) } if builder.cs.sgdIndexer != nil { - dependencies = append(dependencies, [2]blockdao.BlockIndexer{builder.cs.factory, builder.cs.sgdIndexer}) + synchronizedIndexers = append(synchronizedIndexers, builder.cs.sgdIndexer) } - if len(dependencies) > 0 { - dependentIndexer, err := blockindex.NewDependentIndexers(dependencies...) - if err != nil { - return errors.Wrapf(err, "failed to create dependent indexers") - } - indexers = append(indexers, dependentIndexer) + if len(synchronizedIndexers) > 1 { + indexers = append(indexers, blockindex.NewIndexerGroup(synchronizedIndexers...)) } else { indexers = append(indexers, builder.cs.factory) } From 008f35982b9c340859d9757df5a7a348fc83e7c8 Mon Sep 17 00:00:00 2001 From: envestcc Date: Fri, 28 Jul 2023 12:46:35 +0800 Subject: [PATCH 08/13] init startheight when start --- blockindex/indexer_group.go | 70 +++++++++++++++++--------------- blockindex/indexer_group_test.go | 5 +++ 2 files changed, 42 insertions(+), 33 deletions(-) diff --git a/blockindex/indexer_group.go b/blockindex/indexer_group.go index 60beb91a81..28ea8c89de 100644 --- a/blockindex/indexer_group.go +++ b/blockindex/indexer_group.go @@ -15,7 +15,9 @@ import ( // IndexerGroup is a special index that includes multiple indexes, // which stay in sync when blocks are added. type IndexerGroup struct { - indexers []blockdao.BlockIndexer + indexers []blockdao.BlockIndexer + startHeights []uint64 + minStartHeight uint64 } // NewIndexerGroup creates a new indexer group @@ -30,7 +32,7 @@ func (ig *IndexerGroup) Start(ctx context.Context) error { return err } } - return nil + return ig.initStartHeight() } // Stop stops the indexer group @@ -45,16 +47,10 @@ func (ig *IndexerGroup) Stop(ctx context.Context) error { // PutBlock puts a block into the indexers in the group func (ig *IndexerGroup) PutBlock(ctx context.Context, blk *block.Block) error { - for _, indexer := range ig.indexers { - // check if the indexer is a BlockIndexerWithStart - if indexerWithStart, ok := indexer.(blockdao.BlockIndexerWithStart); ok { - startHeight, err := indexerWithStart.StartHeight() - if err != nil { - return err - } - if blk.Height() < startHeight { - continue - } + for i, indexer := range ig.indexers { + // check if the block is higher than the indexer's start height + if blk.Height() < ig.startHeights[i] { + continue } // check if the block is higher than the indexer's height height, err := indexer.Height() @@ -84,27 +80,7 @@ func (ig *IndexerGroup) DeleteTipBlock(ctx context.Context, blk *block.Block) er // StartHeight returns the minimum start height of the indexers in the group func (ig *IndexerGroup) StartHeight() (uint64, error) { - var result uint64 - for i, indexer := range ig.indexers { - tipHeight, err := indexer.Height() - if err != nil { - return 0, err - } - indexStartHeight := tipHeight + 1 - if indexerWithStart, ok := indexer.(blockdao.BlockIndexerWithStart); ok { - startHeight, err := indexerWithStart.StartHeight() - if err != nil { - return 0, err - } - if startHeight > indexStartHeight { - indexStartHeight = startHeight - } - } - if i == 0 || indexStartHeight < result { - result = indexStartHeight - } - } - return result, nil + return ig.minStartHeight, nil } // Height returns the minimum height of the indexers in the group @@ -121,3 +97,31 @@ func (ig *IndexerGroup) Height() (uint64, error) { } return height, nil } + +// initStartHeight initializes the start height of the indexers in the group +// for every indexer, the start height is the maximum of tipheight+1 and startheight +func (ig *IndexerGroup) initStartHeight() error { + ig.minStartHeight = 0 + ig.startHeights = make([]uint64, len(ig.indexers)) + for i, indexer := range ig.indexers { + tipHeight, err := indexer.Height() + if err != nil { + return err + } + indexStartHeight := tipHeight + 1 + if indexerWithStart, ok := indexer.(blockdao.BlockIndexerWithStart); ok { + startHeight, err := indexerWithStart.StartHeight() + if err != nil { + return err + } + if startHeight > indexStartHeight { + indexStartHeight = startHeight + } + } + ig.startHeights[i] = indexStartHeight + if i == 0 || indexStartHeight < ig.minStartHeight { + ig.minStartHeight = indexStartHeight + } + } + return nil +} diff --git a/blockindex/indexer_group_test.go b/blockindex/indexer_group_test.go index 8321130f29..3e91888516 100644 --- a/blockindex/indexer_group_test.go +++ b/blockindex/indexer_group_test.go @@ -6,6 +6,7 @@ package blockindex import ( + "context" "testing" "github.com/golang/mock/gomock" @@ -49,16 +50,20 @@ func TestIndexerGroup_StartHeight(t *testing.T) { for _, indexer := range c.indexers { if indexer[0] > 0 { mockIndexerWithStart := mock_blockdao.NewMockBlockIndexerWithStart(ctrl) + mockIndexerWithStart.EXPECT().Start(gomock.Any()).Return(nil).Times(1) mockIndexerWithStart.EXPECT().StartHeight().Return(indexer[0], nil).Times(1) mockIndexerWithStart.EXPECT().Height().Return(indexer[1], nil).Times(1) indexers = append(indexers, mockIndexerWithStart) } else { mockIndexer := mock_blockdao.NewMockBlockIndexer(ctrl) + mockIndexer.EXPECT().Start(gomock.Any()).Return(nil).Times(1) mockIndexer.EXPECT().Height().Return(indexer[1], nil).Times(1) indexers = append(indexers, mockIndexer) } } ig := NewIndexerGroup(indexers...) + err := ig.Start(context.Background()) + require.NoError(err) height, err := ig.StartHeight() require.NoError(err) require.Equal(c.expect, height) From a9001d700fb4c02f25370d5bef0e0986ed20bd64 Mon Sep 17 00:00:00 2001 From: envestcc Date: Fri, 28 Jul 2023 12:54:19 +0800 Subject: [PATCH 09/13] revert error return in StartHeight --- blockchain/blockdao/blockindexer.go | 7 ++----- blockchain/blockdao/blockindexer_test.go | 2 +- blockindex/contractstaking/indexer.go | 4 ++-- blockindex/contractstaking/indexer_test.go | 4 +--- blockindex/indexer_group.go | 9 +++------ blockindex/indexer_group_test.go | 5 ++--- blockindex/sgd_indexer.go | 4 ++-- blockindex/sgd_indexer_test.go | 4 +--- test/mock/mock_blockdao/mock_blockindexer_withstart.go | 5 ++--- 9 files changed, 16 insertions(+), 28 deletions(-) diff --git a/blockchain/blockdao/blockindexer.go b/blockchain/blockdao/blockindexer.go index 8c0ca6a4b3..dfd70241ae 100644 --- a/blockchain/blockdao/blockindexer.go +++ b/blockchain/blockdao/blockindexer.go @@ -32,7 +32,7 @@ type ( BlockIndexerWithStart interface { BlockIndexer // StartHeight returns the start height of the indexer - StartHeight() (uint64, error) + StartHeight() uint64 } // BlockIndexerChecker defines a checker of block indexer @@ -76,10 +76,7 @@ func (bic *BlockIndexerChecker) CheckIndexer(ctx context.Context, indexer BlockI } startHeight := tipHeight + 1 if indexerWS, ok := indexer.(BlockIndexerWithStart); ok { - indexStartHeight, err := indexerWS.StartHeight() - if err != nil { - return err - } + indexStartHeight := indexerWS.StartHeight() if indexStartHeight > startHeight { startHeight = indexStartHeight } diff --git a/blockchain/blockdao/blockindexer_test.go b/blockchain/blockdao/blockindexer_test.go index 26a11d1241..d80ae9788e 100644 --- a/blockchain/blockdao/blockindexer_test.go +++ b/blockchain/blockdao/blockindexer_test.go @@ -125,7 +125,7 @@ func TestCheckIndexerWithStart(t *testing.T) { }).AnyTimes() mockDao.EXPECT().GetReceipts(gomock.Any()).Return(nil, nil).AnyTimes() indexer.EXPECT().Height().Return(c.indexerTipHeight, nil).Times(1) - indexer.EXPECT().StartHeight().Return(c.indexerStartHeight, nil).AnyTimes() + indexer.EXPECT().StartHeight().Return(c.indexerStartHeight).AnyTimes() indexer.EXPECT().PutBlock(gomock.Any(), gomock.Any()).DoAndReturn(func(arg0 context.Context, arg1 *block.Block) error { putBlocks = append(putBlocks, arg1) return nil diff --git a/blockindex/contractstaking/indexer.go b/blockindex/contractstaking/indexer.go index 516217a1c3..5adcf8a386 100644 --- a/blockindex/contractstaking/indexer.go +++ b/blockindex/contractstaking/indexer.go @@ -75,8 +75,8 @@ func (s *Indexer) Height() (uint64, error) { } // StartHeight returns the start height of the indexer -func (s *Indexer) StartHeight() (uint64, error) { - return s.contractDeployHeight, nil +func (s *Indexer) StartHeight() uint64 { + return s.contractDeployHeight } // CandidateVotes returns the candidate votes diff --git a/blockindex/contractstaking/indexer_test.go b/blockindex/contractstaking/indexer_test.go index efa1ab17ca..20a61cc885 100644 --- a/blockindex/contractstaking/indexer_test.go +++ b/blockindex/contractstaking/indexer_test.go @@ -101,9 +101,7 @@ func TestContractStakingIndexerLoadCache(t *testing.T) { newHeight, err := newIndexer.Height() r.NoError(err) r.Equal(height, newHeight) - newStartHeight, err := newIndexer.StartHeight() - r.NoError(err) - r.Equal(startHeight, newStartHeight) + r.Equal(startHeight, newIndexer.StartHeight()) r.EqualValues(1, newIndexer.TotalBucketCount()) r.NoError(newIndexer.Stop(context.Background())) } diff --git a/blockindex/indexer_group.go b/blockindex/indexer_group.go index 28ea8c89de..272c6484c6 100644 --- a/blockindex/indexer_group.go +++ b/blockindex/indexer_group.go @@ -79,8 +79,8 @@ func (ig *IndexerGroup) DeleteTipBlock(ctx context.Context, blk *block.Block) er } // StartHeight returns the minimum start height of the indexers in the group -func (ig *IndexerGroup) StartHeight() (uint64, error) { - return ig.minStartHeight, nil +func (ig *IndexerGroup) StartHeight() uint64 { + return ig.minStartHeight } // Height returns the minimum height of the indexers in the group @@ -110,10 +110,7 @@ func (ig *IndexerGroup) initStartHeight() error { } indexStartHeight := tipHeight + 1 if indexerWithStart, ok := indexer.(blockdao.BlockIndexerWithStart); ok { - startHeight, err := indexerWithStart.StartHeight() - if err != nil { - return err - } + startHeight := indexerWithStart.StartHeight() if startHeight > indexStartHeight { indexStartHeight = startHeight } diff --git a/blockindex/indexer_group_test.go b/blockindex/indexer_group_test.go index 3e91888516..5760140c26 100644 --- a/blockindex/indexer_group_test.go +++ b/blockindex/indexer_group_test.go @@ -51,7 +51,7 @@ func TestIndexerGroup_StartHeight(t *testing.T) { if indexer[0] > 0 { mockIndexerWithStart := mock_blockdao.NewMockBlockIndexerWithStart(ctrl) mockIndexerWithStart.EXPECT().Start(gomock.Any()).Return(nil).Times(1) - mockIndexerWithStart.EXPECT().StartHeight().Return(indexer[0], nil).Times(1) + mockIndexerWithStart.EXPECT().StartHeight().Return(indexer[0]).Times(1) mockIndexerWithStart.EXPECT().Height().Return(indexer[1], nil).Times(1) indexers = append(indexers, mockIndexerWithStart) } else { @@ -64,8 +64,7 @@ func TestIndexerGroup_StartHeight(t *testing.T) { ig := NewIndexerGroup(indexers...) err := ig.Start(context.Background()) require.NoError(err) - height, err := ig.StartHeight() - require.NoError(err) + height := ig.StartHeight() require.Equal(c.expect, height) }) } diff --git a/blockindex/sgd_indexer.go b/blockindex/sgd_indexer.go index 16bc0c0191..22cd472b78 100644 --- a/blockindex/sgd_indexer.go +++ b/blockindex/sgd_indexer.go @@ -279,8 +279,8 @@ func (sgd *sgdRegistry) Height() (uint64, error) { } // StartHeight returns the start height of the indexer -func (sgd *sgdRegistry) StartHeight() (uint64, error) { - return sgd.startHeight, nil +func (sgd *sgdRegistry) StartHeight() uint64 { + return sgd.startHeight } // PutBlock puts a block into SGDIndexer diff --git a/blockindex/sgd_indexer_test.go b/blockindex/sgd_indexer_test.go index db24a612ff..02ff4e908a 100644 --- a/blockindex/sgd_indexer_test.go +++ b/blockindex/sgd_indexer_test.go @@ -55,9 +55,7 @@ func TestNewSGDRegistry(t *testing.T) { }() nonce := uint64(0) - startHeight, err := sgdRegistry.StartHeight() - r.NoError(err) - r.Equal(nonce, startHeight) + r.Equal(nonce, sgdRegistry.StartHeight()) hh, err := sgdRegistry.Height() r.NoError(err) r.Equal(nonce, hh) diff --git a/test/mock/mock_blockdao/mock_blockindexer_withstart.go b/test/mock/mock_blockdao/mock_blockindexer_withstart.go index e0d16c0eaf..2f8a029269 100644 --- a/test/mock/mock_blockdao/mock_blockindexer_withstart.go +++ b/test/mock/mock_blockdao/mock_blockindexer_withstart.go @@ -93,12 +93,11 @@ func (mr *MockBlockIndexerWithStartMockRecorder) Start(arg0 interface{}) *gomock } // StartHeight mocks base method. -func (m *MockBlockIndexerWithStart) StartHeight() (uint64, error) { +func (m *MockBlockIndexerWithStart) StartHeight() uint64 { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "StartHeight") ret0, _ := ret[0].(uint64) - ret1, _ := ret[1].(error) - return ret0, ret1 + return ret0 } // StartHeight indicates an expected call of StartHeight. From dddbfa831d1fc9249716ebd3d12d1e885608912d Mon Sep 17 00:00:00 2001 From: envestcc Date: Mon, 31 Jul 2023 12:21:54 +0800 Subject: [PATCH 10/13] address comments --- .../{indexer_group.go => sync_indexers.go} | 29 ++++++++++--------- ...er_group_test.go => sync_indexers_test.go} | 4 +-- chainservice/builder.go | 4 +-- 3 files changed, 19 insertions(+), 18 deletions(-) rename blockindex/{indexer_group.go => sync_indexers.go} (75%) rename blockindex/{indexer_group_test.go => sync_indexers_test.go} (97%) diff --git a/blockindex/indexer_group.go b/blockindex/sync_indexers.go similarity index 75% rename from blockindex/indexer_group.go rename to blockindex/sync_indexers.go index 272c6484c6..7324755db5 100644 --- a/blockindex/indexer_group.go +++ b/blockindex/sync_indexers.go @@ -12,21 +12,22 @@ import ( "github.com/iotexproject/iotex-core/blockchain/blockdao" ) -// IndexerGroup is a special index that includes multiple indexes, +// SyncIndexers is a special index that includes multiple indexes, // which stay in sync when blocks are added. -type IndexerGroup struct { +type SyncIndexers struct { indexers []blockdao.BlockIndexer - startHeights []uint64 - minStartHeight uint64 + startHeights []uint64 // start height of each indexer, which will be determined when the indexer is started + minStartHeight uint64 // minimum start height of all indexers } -// NewIndexerGroup creates a new indexer group -func NewIndexerGroup(indexers ...blockdao.BlockIndexer) *IndexerGroup { - return &IndexerGroup{indexers: indexers} +// NewSyncIndexers creates a new SyncIndexers +// each indexer will PutBlock one by one in the order of the indexers +func NewSyncIndexers(indexers ...blockdao.BlockIndexer) *SyncIndexers { + return &SyncIndexers{indexers: indexers} } // Start starts the indexer group -func (ig *IndexerGroup) Start(ctx context.Context) error { +func (ig *SyncIndexers) Start(ctx context.Context) error { for _, indexer := range ig.indexers { if err := indexer.Start(ctx); err != nil { return err @@ -36,7 +37,7 @@ func (ig *IndexerGroup) Start(ctx context.Context) error { } // Stop stops the indexer group -func (ig *IndexerGroup) Stop(ctx context.Context) error { +func (ig *SyncIndexers) Stop(ctx context.Context) error { for _, indexer := range ig.indexers { if err := indexer.Stop(ctx); err != nil { return err @@ -46,7 +47,7 @@ func (ig *IndexerGroup) Stop(ctx context.Context) error { } // PutBlock puts a block into the indexers in the group -func (ig *IndexerGroup) PutBlock(ctx context.Context, blk *block.Block) error { +func (ig *SyncIndexers) PutBlock(ctx context.Context, blk *block.Block) error { for i, indexer := range ig.indexers { // check if the block is higher than the indexer's start height if blk.Height() < ig.startHeights[i] { @@ -69,7 +70,7 @@ func (ig *IndexerGroup) PutBlock(ctx context.Context, blk *block.Block) error { } // DeleteTipBlock deletes the tip block from the indexers in the group -func (ig *IndexerGroup) DeleteTipBlock(ctx context.Context, blk *block.Block) error { +func (ig *SyncIndexers) DeleteTipBlock(ctx context.Context, blk *block.Block) error { for _, indexer := range ig.indexers { if err := indexer.DeleteTipBlock(ctx, blk); err != nil { return err @@ -79,12 +80,12 @@ func (ig *IndexerGroup) DeleteTipBlock(ctx context.Context, blk *block.Block) er } // StartHeight returns the minimum start height of the indexers in the group -func (ig *IndexerGroup) StartHeight() uint64 { +func (ig *SyncIndexers) StartHeight() uint64 { return ig.minStartHeight } // Height returns the minimum height of the indexers in the group -func (ig *IndexerGroup) Height() (uint64, error) { +func (ig *SyncIndexers) Height() (uint64, error) { var height uint64 for i, indexer := range ig.indexers { h, err := indexer.Height() @@ -100,7 +101,7 @@ func (ig *IndexerGroup) Height() (uint64, error) { // initStartHeight initializes the start height of the indexers in the group // for every indexer, the start height is the maximum of tipheight+1 and startheight -func (ig *IndexerGroup) initStartHeight() error { +func (ig *SyncIndexers) initStartHeight() error { ig.minStartHeight = 0 ig.startHeights = make([]uint64, len(ig.indexers)) for i, indexer := range ig.indexers { diff --git a/blockindex/indexer_group_test.go b/blockindex/sync_indexers_test.go similarity index 97% rename from blockindex/indexer_group_test.go rename to blockindex/sync_indexers_test.go index 5760140c26..02734a02d0 100644 --- a/blockindex/indexer_group_test.go +++ b/blockindex/sync_indexers_test.go @@ -16,7 +16,7 @@ import ( "github.com/iotexproject/iotex-core/test/mock/mock_blockdao" ) -func TestIndexerGroup_StartHeight(t *testing.T) { +func TestSyncIndexers_StartHeight(t *testing.T) { require := require.New(t) ctrl := gomock.NewController(t) @@ -61,7 +61,7 @@ func TestIndexerGroup_StartHeight(t *testing.T) { indexers = append(indexers, mockIndexer) } } - ig := NewIndexerGroup(indexers...) + ig := NewSyncIndexers(indexers...) err := ig.Start(context.Background()) require.NoError(err) height := ig.StartHeight() diff --git a/chainservice/builder.go b/chainservice/builder.go index c64b611b94..3aaa189631 100644 --- a/chainservice/builder.go +++ b/chainservice/builder.go @@ -257,7 +257,7 @@ func (builder *Builder) buildBlockDAO(forTest bool) error { } var indexers []blockdao.BlockIndexer - // indexers in synchronizedIndexers will be putblock one by one for every block + // indexers in synchronizedIndexers will need to run PutBlock() one by one synchronizedIndexers := []blockdao.BlockIndexer{builder.cs.factory} if builder.cs.contractStakingIndexer != nil { synchronizedIndexers = append(synchronizedIndexers, builder.cs.contractStakingIndexer) @@ -266,7 +266,7 @@ func (builder *Builder) buildBlockDAO(forTest bool) error { synchronizedIndexers = append(synchronizedIndexers, builder.cs.sgdIndexer) } if len(synchronizedIndexers) > 1 { - indexers = append(indexers, blockindex.NewIndexerGroup(synchronizedIndexers...)) + indexers = append(indexers, blockindex.NewSyncIndexers(synchronizedIndexers...)) } else { indexers = append(indexers, builder.cs.factory) } From 682f5b02538cecc9c3c49e8383d8ea201ace33bc Mon Sep 17 00:00:00 2001 From: envestcc Date: Mon, 31 Jul 2023 21:35:51 +0800 Subject: [PATCH 11/13] add tests --- blockindex/sync_indexers_test.go | 146 +++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/blockindex/sync_indexers_test.go b/blockindex/sync_indexers_test.go index 02734a02d0..62daa6c85b 100644 --- a/blockindex/sync_indexers_test.go +++ b/blockindex/sync_indexers_test.go @@ -7,12 +7,15 @@ package blockindex import ( "context" + "strconv" "testing" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + "github.com/iotexproject/iotex-core/blockchain/block" "github.com/iotexproject/iotex-core/blockchain/blockdao" + "github.com/iotexproject/iotex-core/test/identityset" "github.com/iotexproject/iotex-core/test/mock/mock_blockdao" ) @@ -70,3 +73,146 @@ func TestSyncIndexers_StartHeight(t *testing.T) { } } + +func TestSyncIndexers_Height(t *testing.T) { + require := require.New(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + cases := []struct { + heights []uint64 + expect uint64 + }{ + {[]uint64{}, 0}, + {[]uint64{100}, 100}, + {[]uint64{100, 100}, 100}, + {[]uint64{90, 100}, 90}, + {[]uint64{100, 90}, 90}, + {[]uint64{100, 100, 100}, 100}, + {[]uint64{90, 100, 100}, 90}, + {[]uint64{90, 80, 100}, 80}, + {[]uint64{90, 80, 70}, 70}, + } + + for i := range cases { + name := strconv.FormatUint(uint64(i), 10) + t.Run(name, func(t *testing.T) { + var indexers []blockdao.BlockIndexer + for _, height := range cases[i].heights { + mockIndexer := mock_blockdao.NewMockBlockIndexer(ctrl) + mockIndexer.EXPECT().Height().Return(height, nil).Times(1) + indexers = append(indexers, mockIndexer) + } + ig := NewSyncIndexers(indexers...) + height, err := ig.Height() + require.NoError(err) + require.Equal(cases[i].expect, height) + }) + } +} + +func TestSyncIndexers_PutBlock(t *testing.T) { + require := require.New(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + cases := []struct { + indexers [][2]uint64 // [startHeight, height] + blocks []uint64 // blocks to put + expectBlocks [][]uint64 // expect blocks to put on every indexer + }{ + { + [][2]uint64{}, + []uint64{100}, + [][]uint64{}, + }, + { + [][2]uint64{{100, 10}}, + []uint64{10, 20, 90, 100, 101}, + [][]uint64{{100, 101}}, + }, + { + [][2]uint64{{100, 210}}, + []uint64{10, 20, 90, 100, 101, 210, 211}, + [][]uint64{{211}}, + }, + { + [][2]uint64{{0, 200}, {250, 1}}, + []uint64{1, 2, 201, 249, 250, 251}, + [][]uint64{{201, 249, 250, 251}, {250, 251}}, + }, + { + [][2]uint64{{0, 250}, {250, 250}}, + []uint64{1, 2, 201, 249, 250, 251, 252}, + [][]uint64{{251, 252}, {251, 252}}, + }, + { + [][2]uint64{{0, 200}, {250, 1}, {300, 1}}, + []uint64{1, 2, 201, 249, 250, 251, 300, 301}, + [][]uint64{{201, 249, 250, 251, 300, 301}, {250, 251, 300, 301}, {300, 301}}, + }, + { + [][2]uint64{{0, 250}, {250, 250}, {300, 250}}, + []uint64{1, 2, 201, 249, 250, 251, 300, 301}, + [][]uint64{{251, 300, 301}, {251, 300, 301}, {300, 301}}, + }, + { + [][2]uint64{{0, 300}, {250, 300}, {300, 300}}, + []uint64{1, 2, 201, 249, 250, 251, 300, 301}, + [][]uint64{{301}, {301}, {301}}, + }, + { + [][2]uint64{{0, 400}, {250, 400}, {300, 400}}, + []uint64{1, 2, 201, 249, 250, 251, 300, 301, 400, 401}, + [][]uint64{{401}, {401}, {401}}, + }, + } + + for _, c := range cases { + t.Run("", func(t *testing.T) { + var indexers []blockdao.BlockIndexer + putBlocks := make([][]uint64, len(c.indexers)) + indexersHeight := make([]uint64, len(c.indexers)) + for id, indexer := range c.indexers { + idx := id + indexersHeight[idx] = indexer[1] + if indexer[0] > 0 { + mockIndexerWithStart := mock_blockdao.NewMockBlockIndexerWithStart(ctrl) + mockIndexerWithStart.EXPECT().Start(gomock.Any()).Return(nil).Times(1) + mockIndexerWithStart.EXPECT().StartHeight().Return(indexer[0]).Times(1) + mockIndexerWithStart.EXPECT().Height().DoAndReturn(func() (uint64, error) { + return indexersHeight[idx], nil + }).AnyTimes() + mockIndexerWithStart.EXPECT().PutBlock(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, blk *block.Block) error { + putBlocks[idx] = append(putBlocks[idx], blk.Height()) + indexersHeight[idx] = blk.Height() + return nil + }).Times(len(c.expectBlocks[idx])) + indexers = append(indexers, mockIndexerWithStart) + } else { + mockIndexer := mock_blockdao.NewMockBlockIndexer(ctrl) + mockIndexer.EXPECT().Start(gomock.Any()).Return(nil).Times(1) + mockIndexer.EXPECT().Height().DoAndReturn(func() (uint64, error) { + return indexersHeight[idx], nil + }).AnyTimes() + mockIndexer.EXPECT().PutBlock(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, blk *block.Block) error { + putBlocks[idx] = append(putBlocks[idx], blk.Height()) + indexersHeight[idx] = blk.Height() + return nil + }).Times(len(c.expectBlocks[idx])) + indexers = append(indexers, mockIndexer) + } + } + ig := NewSyncIndexers(indexers...) + err := ig.Start(context.Background()) + require.NoError(err) + for _, blkHeight := range c.blocks { + blk, err := block.NewBuilder(block.RunnableActions{}).SetHeight(blkHeight).SignAndBuild(identityset.PrivateKey(0)) + require.NoError(err) + err = ig.PutBlock(context.Background(), &blk) + require.NoError(err) + } + require.Equal(c.expectBlocks, putBlocks) + }) + } +} From ca82ab88a11a9eb193ae64ff7888ca75cdfcd8d4 Mon Sep 17 00:00:00 2001 From: envestcc Date: Tue, 1 Aug 2023 11:22:08 +0800 Subject: [PATCH 12/13] remove unused dependency --- go.mod | 1 - go.sum | 1 - 2 files changed, 2 deletions(-) diff --git a/go.mod b/go.mod index 8b387cbb89..0447c22750 100644 --- a/go.mod +++ b/go.mod @@ -63,7 +63,6 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.34.0 golang.org/x/exp v0.0.0-20230206171751-46f607a40771 golang.org/x/text v0.7.0 - gonum.org/v1/gonum v0.6.0 ) require ( diff --git a/go.sum b/go.sum index 5e7c83e631..a065dab82b 100644 --- a/go.sum +++ b/go.sum @@ -1624,7 +1624,6 @@ golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= golang.org/x/xerrors v0.0.0-20190212162355-a5947ffaace3/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.6.0 h1:DJy6UzXbahnGUf1ujUNkh/NEtK14qMo2nvlBPs4U5yw= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= From d7d5d06a09b897838126ceb53fe193c67612d137 Mon Sep 17 00:00:00 2001 From: envestcc Date: Wed, 2 Aug 2023 14:25:13 +0800 Subject: [PATCH 13/13] add comments --- chainservice/builder.go | 1 + 1 file changed, 1 insertion(+) diff --git a/chainservice/builder.go b/chainservice/builder.go index 3aaa189631..78d7c78adc 100644 --- a/chainservice/builder.go +++ b/chainservice/builder.go @@ -258,6 +258,7 @@ func (builder *Builder) buildBlockDAO(forTest bool) error { var indexers []blockdao.BlockIndexer // indexers in synchronizedIndexers will need to run PutBlock() one by one + // factory is dependent on sgdIndexer and contractStakingIndexer, so it should be put in the first place synchronizedIndexers := []blockdao.BlockIndexer{builder.cs.factory} if builder.cs.contractStakingIndexer != nil { synchronizedIndexers = append(synchronizedIndexers, builder.cs.contractStakingIndexer)