From f7a23355e91eca484a7cf90e03e2dd241291ad56 Mon Sep 17 00:00:00 2001 From: huangyi Date: Thu, 12 Sep 2024 10:28:14 +0800 Subject: [PATCH 1/5] Problem: pre-estimation don't run in parallel --- app/executor.go | 105 +++++++++++++++++++++++++++++++----------------- 1 file changed, 68 insertions(+), 37 deletions(-) diff --git a/app/executor.go b/app/executor.go index 8cbbaafc58..18b830c23b 100644 --- a/app/executor.go +++ b/app/executor.go @@ -3,6 +3,7 @@ package app import ( "context" "io" + "sync" "sync/atomic" "cosmossdk.io/collections" @@ -21,6 +22,8 @@ import ( blockstm "github.com/crypto-org-chain/go-block-stm" ) +const MinimalParallelPreEstimate = 32 + func DefaultTxExecutor(_ context.Context, txs [][]byte, ms storetypes.MultiStore, @@ -73,17 +76,14 @@ func STMTxExecutor( incarnationCache[i].Store(&m) } - var estimates map[int]blockstm.MultiLocations - memTxs := make([]sdk.Tx, len(txs)) + var ( + estimates map[int]blockstm.MultiLocations + memTxs []sdk.Tx + ) if estimate { - for i, rawTx := range txs { - if memTx, err := txDecoder(rawTx); err == nil { - memTxs[i] = memTx - } - } // pre-estimation evmDenom := evmKeeper.GetParams(sdk.NewContext(ms, cmtproto.Header{}, false, log.NewNopLogger())).EvmDenom - estimates = preEstimates(memTxs, authStore, bankStore, evmDenom) + memTxs, estimates = preEstimates(txs, workers, authStore, bankStore, evmDenom) } if err := blockstm.ExecuteBlockWithEstimates( @@ -188,40 +188,71 @@ func (ms stmMultiStoreWrapper) GetObjKVStore(key storetypes.StoreKey) storetypes // preEstimates returns a static estimation of the written keys for each transaction. // NOTE: make sure it sync with the latest sdk logic when sdk upgrade. -func preEstimates(txs []sdk.Tx, authStore, bankStore int, evmDenom string) map[int]blockstm.MultiLocations { +func preEstimates(txs [][]byte, workers, authStore, bankStore int, evmDenom string, txDecoder sdk.TxDecoder) ([]sdk.Tx, map[int]blockstm.MultiLocations) { + memTxs := make([]sdk.Tx, len(txs)) estimates := make(map[int]blockstm.MultiLocations, len(txs)) - for i, tx := range txs { - feeTx, ok := tx.(sdk.FeeTx) - if !ok { - continue - } - feePayer := sdk.AccAddress(feeTx.FeePayer()) - // account key - accKey, err := collections.EncodeKeyWithPrefix( - authtypes.AddressStoreKeyPrefix, - sdk.AccAddressKey, - feePayer, - ) - if err != nil { - continue - } + job := func(start, end int) { + for i := start; i < end; i++ { + rawTx := txs[i] + tx, err := txDecoder(rawTx) + if err != nil { + continue + } + memTxs[i] = tx - // balance key - balanceKey, err := collections.EncodeKeyWithPrefix( - banktypes.BalancesPrefix, - collections.PairKeyCodec(sdk.AccAddressKey, collections.StringKey), - collections.Join(feePayer, evmDenom), - ) - if err != nil { - continue - } + feeTx, ok := tx.(sdk.FeeTx) + if !ok { + continue + } + feePayer := sdk.AccAddress(feeTx.FeePayer()) + + // account key + accKey, err := collections.EncodeKeyWithPrefix( + authtypes.AddressStoreKeyPrefix, + sdk.AccAddressKey, + feePayer, + ) + if err != nil { + continue + } - estimates[i] = blockstm.MultiLocations{ - authStore: {accKey}, - bankStore: {balanceKey}, + // balance key + balanceKey, err := collections.EncodeKeyWithPrefix( + banktypes.BalancesPrefix, + collections.PairKeyCodec(sdk.AccAddressKey, collections.StringKey), + collections.Join(feePayer, evmDenom), + ) + if err != nil { + continue + } + + estimates[i] = blockstm.MultiLocations{ + authStore: {accKey}, + bankStore: {balanceKey}, + } } } - return estimates + blockSize := len(txs) + chunk := blockSize / workers + if blockSize < MinimalParallelPreEstimate || chunk == 0 { + job(0, blockSize) + } else { + var wg sync.WaitGroup + wg.Add(workers) + for i := 0; i < workers; i++ { + start := i * chunk + end := (i + 1) * chunk + if i == workers-1 { + end = blockSize + } + go func() { + defer wg.Done() + job(start, end) + }() + } + wg.Wait() + } + return memTxs, estimates } From 213f52ebf38c5897bf19ff237682d5dd2f05f7d9 Mon Sep 17 00:00:00 2001 From: huangyi Date: Thu, 12 Sep 2024 10:30:09 +0800 Subject: [PATCH 2/5] fix build --- app/executor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/executor.go b/app/executor.go index 18b830c23b..fda0d6bd8f 100644 --- a/app/executor.go +++ b/app/executor.go @@ -83,7 +83,7 @@ func STMTxExecutor( if estimate { // pre-estimation evmDenom := evmKeeper.GetParams(sdk.NewContext(ms, cmtproto.Header{}, false, log.NewNopLogger())).EvmDenom - memTxs, estimates = preEstimates(txs, workers, authStore, bankStore, evmDenom) + memTxs, estimates = preEstimates(txs, workers, authStore, bankStore, evmDenom, txDecoder) } if err := blockstm.ExecuteBlockWithEstimates( From 99ca91e4beaeaeac98b74eb9d3c2022de774892b Mon Sep 17 00:00:00 2001 From: huangyi Date: Thu, 12 Sep 2024 10:52:53 +0800 Subject: [PATCH 3/5] fix race --- app/executor.go | 12 ++++++++---- encoding/config.go | 9 ++++++++- go.mod | 2 +- go.sum | 4 ++-- gomod2nix.toml | 4 ++-- 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/app/executor.go b/app/executor.go index fda0d6bd8f..89d28d2d57 100644 --- a/app/executor.go +++ b/app/executor.go @@ -77,7 +77,7 @@ func STMTxExecutor( } var ( - estimates map[int]blockstm.MultiLocations + estimates []blockstm.MultiLocations memTxs []sdk.Tx ) if estimate { @@ -103,7 +103,11 @@ func STMTxExecutor( cache = *v } - results[txn] = deliverTxWithMultiStore(int(txn), memTxs[txn], msWrapper{ms}, cache) + var memTx sdk.Tx + if memTxs != nil { + memTx = memTxs[txn] + } + results[txn] = deliverTxWithMultiStore(int(txn), memTx, msWrapper{ms}, cache) if v != nil { incarnationCache[txn].Store(v) @@ -188,9 +192,9 @@ func (ms stmMultiStoreWrapper) GetObjKVStore(key storetypes.StoreKey) storetypes // preEstimates returns a static estimation of the written keys for each transaction. // NOTE: make sure it sync with the latest sdk logic when sdk upgrade. -func preEstimates(txs [][]byte, workers, authStore, bankStore int, evmDenom string, txDecoder sdk.TxDecoder) ([]sdk.Tx, map[int]blockstm.MultiLocations) { +func preEstimates(txs [][]byte, workers, authStore, bankStore int, evmDenom string, txDecoder sdk.TxDecoder) ([]sdk.Tx, []blockstm.MultiLocations) { memTxs := make([]sdk.Tx, len(txs)) - estimates := make(map[int]blockstm.MultiLocations, len(txs)) + estimates := make([]blockstm.MultiLocations, len(txs)) job := func(start, end int) { for i := start; i < end; i++ { diff --git a/encoding/config.go b/encoding/config.go index faf4406d18..87bc692175 100644 --- a/encoding/config.go +++ b/encoding/config.go @@ -50,10 +50,17 @@ func MakeConfig() ethermint.EncodingConfig { panic(err) } codec := amino.NewProtoCodec(interfaceRegistry) + txConfig, err := tx.NewTxConfigWithOptions(codec, tx.ConfigOptions{ + EnabledSignModes: tx.DefaultSignModes, + SigningContext: interfaceRegistry.SigningContext(), + }) + if err != nil { + panic(err) + } encodingConfig := ethermint.EncodingConfig{ InterfaceRegistry: interfaceRegistry, Codec: codec, - TxConfig: tx.NewTxConfig(codec, tx.DefaultSignModes), + TxConfig: txConfig, Amino: cdc, } enccodec.RegisterLegacyAminoCodec(cdc) diff --git a/go.mod b/go.mod index 4724128218..3c0f63db35 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/cosmos/ibc-go/modules/capability v1.0.0 github.com/cosmos/ibc-go/v8 v8.1.0 github.com/cosmos/rosetta v0.50.3-1 - github.com/crypto-org-chain/go-block-stm v0.0.0-20240911081142-92839e79a3ae + github.com/crypto-org-chain/go-block-stm v0.0.0-20240912024944-1cd89976aa5e github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/ethereum/go-ethereum v1.10.26 github.com/gogo/protobuf v1.3.2 diff --git a/go.sum b/go.sum index 3f98d733d4..5a1a9c20dc 100644 --- a/go.sum +++ b/go.sum @@ -419,8 +419,8 @@ github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240911084450-6870ba130be2 github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240911084450-6870ba130be2/go.mod h1:gjE3DZe4t/+VeIk6CmrouyqiuDbZ7QOVDDq3nLqBTpg= github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240911084450-6870ba130be2 h1:mxlOSCru7YgmX055rrlkCSUu0D8lAqJ8Dnhp0yXCBuM= github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240911084450-6870ba130be2/go.mod h1:RTiTs4hkXG6IvYGknvB8p79YgjYJdcbzLUOGJChsPnY= -github.com/crypto-org-chain/go-block-stm v0.0.0-20240911081142-92839e79a3ae h1:gakWYsVubWX8P9NpxaPnvg0UJYfIZigfko5WN57t7OA= -github.com/crypto-org-chain/go-block-stm v0.0.0-20240911081142-92839e79a3ae/go.mod h1:iwQTX9xMX8NV9k3o2BiWXA0SswpsZrDk5q3gA7nWYiE= +github.com/crypto-org-chain/go-block-stm v0.0.0-20240912024944-1cd89976aa5e h1:FFpE6+Y4o5GxkeGwUcETM6amgohh7msWvWf1MDqueVc= +github.com/crypto-org-chain/go-block-stm v0.0.0-20240912024944-1cd89976aa5e/go.mod h1:iwQTX9xMX8NV9k3o2BiWXA0SswpsZrDk5q3gA7nWYiE= github.com/crypto-org-chain/go-ethereum v1.10.20-0.20240425065928-ebb09502e7a7 h1:V43F3JFcqG4MUThf9W/DytnPblpR6CcaLBw2Wx6zTgE= github.com/crypto-org-chain/go-ethereum v1.10.20-0.20240425065928-ebb09502e7a7/go.mod h1:+a8pUj1tOyJ2RinsNQD4326YS+leSoKGiG/uVVb0x6Y= github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE= diff --git a/gomod2nix.toml b/gomod2nix.toml index 3c1044e75d..aafb878fe2 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -203,8 +203,8 @@ schema = 3 version = "v0.0.24" hash = "sha256-4vUukHONOjNn0qfQr4esK6TWfPWsIp+rbdz65og84lw=" [mod."github.com/crypto-org-chain/go-block-stm"] - version = "v0.0.0-20240911081142-92839e79a3ae" - hash = "sha256-8MhSeC5BB5BwOet3k3Rfua0TBeBATtLSwaW6s5WpYCM=" + version = "v0.0.0-20240912024944-1cd89976aa5e" + hash = "sha256-rY8W4dSciOXT29MCySbH5sw0Fp15IQVgBK9QlMX0JeU=" [mod."github.com/danieljoos/wincred"] version = "v1.2.0" hash = "sha256-LHcvTJCc8++bFndbd8ZgMSTe4L5h2C4rN+cSWHCz54Y=" From ac898984e4a974d48060581950aec7f093e5d5ff Mon Sep 17 00:00:00 2001 From: huangyi Date: Thu, 12 Sep 2024 11:57:12 +0800 Subject: [PATCH 4/5] cleanup chunking --- app/executor.go | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/app/executor.go b/app/executor.go index 89d28d2d57..1cff225d61 100644 --- a/app/executor.go +++ b/app/executor.go @@ -22,7 +22,7 @@ import ( blockstm "github.com/crypto-org-chain/go-block-stm" ) -const MinimalParallelPreEstimate = 32 +const MinimalParallelPreEstimate = 16 func DefaultTxExecutor(_ context.Context, txs [][]byte, @@ -239,24 +239,18 @@ func preEstimates(txs [][]byte, workers, authStore, bankStore int, evmDenom stri } blockSize := len(txs) - chunk := blockSize / workers - if blockSize < MinimalParallelPreEstimate || chunk == 0 { - job(0, blockSize) - } else { - var wg sync.WaitGroup - wg.Add(workers) - for i := 0; i < workers; i++ { - start := i * chunk - end := (i + 1) * chunk - if i == workers-1 { - end = blockSize - } - go func() { - defer wg.Done() - job(start, end) - }() - } - wg.Wait() + chunk := (blockSize + workers - 1) / workers + var wg sync.WaitGroup + for i := 0; i < blockSize; i += chunk { + start := i + end := min(i+chunk, blockSize) + wg.Add(1) + go func() { + defer wg.Done() + job(start, end) + }() } + wg.Wait() + return memTxs, estimates } From 6629d4e310da40b251d7d76e37b64150b6440531 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Thu, 12 Sep 2024 12:03:40 +0800 Subject: [PATCH 5/5] keep unchanged --- encoding/config.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/encoding/config.go b/encoding/config.go index 87bc692175..faf4406d18 100644 --- a/encoding/config.go +++ b/encoding/config.go @@ -50,17 +50,10 @@ func MakeConfig() ethermint.EncodingConfig { panic(err) } codec := amino.NewProtoCodec(interfaceRegistry) - txConfig, err := tx.NewTxConfigWithOptions(codec, tx.ConfigOptions{ - EnabledSignModes: tx.DefaultSignModes, - SigningContext: interfaceRegistry.SigningContext(), - }) - if err != nil { - panic(err) - } encodingConfig := ethermint.EncodingConfig{ InterfaceRegistry: interfaceRegistry, Codec: codec, - TxConfig: txConfig, + TxConfig: tx.NewTxConfig(codec, tx.DefaultSignModes), Amino: cdc, } enccodec.RegisterLegacyAminoCodec(cdc)