Problem: block-stm tx executor bad worst case performance (#522)

* Problem: block-stm tx executor don't do simple static dependency
analysis

Solution:
- estimate dependencies based on tx fee payer, try to optimise worst case
  performance.

* fix build

* customize estimates instead of dependencies

* fix context

* update api

* cleanup

* changelog

* cleanup

* cleanup

* cleanup

* pre-estimate config

* fix lint

CHANGELOG.md

@@ -87,6 +87,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * (deps) [#505](https://github.com/crypto-org-chain/ethermint/pull/505) Update cometbft to v0.38.10.
 * (ante) [#504](https://github.com/crypto-org-chain/ethermint/pull/504) Optimize AnteHandle method to skip checks if disabledMsgs is empty.
 * [#517](https://github.com/crypto-org-chain/ethermint/pull/517) Add check for integer overflow to ensure safe conversion.
+* [#522](https://github.com/crypto-org-chain/ethermint/pull/522) block-stm executor support optional pre-estimations.
 
 ## v0.21.x-cronos
 

app/app.go

@@ -337,18 +337,6 @@ func NewEthermintApp(
 		okeys:             okeys,
 	}
 
-	executor := cast.ToString(appOpts.Get(srvflags.EVMBlockExecutor))
-	switch executor {
-	case srvconfig.BlockExecutorBlockSTM:
-		sdk.SetAddrCacheEnabled(false)
-		workers := cast.ToInt(appOpts.Get(srvflags.EVMBlockSTMWorkers))
-		app.SetTxExecutor(STMTxExecutor(app.GetStoreKeys(), workers))
-	case "", srvconfig.BlockExecutorSequential:
-		app.SetTxExecutor(DefaultTxExecutor)
-	default:
-		panic(fmt.Errorf("unknown EVM block executor: %s", executor))
-	}
-
 	// init params keeper and subspaces
 	app.ParamsKeeper = initParamsKeeper(appCodec, cdc, keys[paramstypes.StoreKey], tkeys[paramstypes.TStoreKey])
 
@@ -803,6 +791,19 @@ func NewEthermintApp(
 	app.ScopedIBCKeeper = scopedIBCKeeper
 	app.ScopedTransferKeeper = scopedTransferKeeper
 
+	executor := cast.ToString(appOpts.Get(srvflags.EVMBlockExecutor))
+	switch executor {
+	case srvconfig.BlockExecutorBlockSTM:
+		sdk.SetAddrCacheEnabled(false)
+		workers := cast.ToInt(appOpts.Get(srvflags.EVMBlockSTMWorkers))
+		preEstimate := cast.ToBool(appOpts.Get(srvflags.EVMBlockSTMPreEstimate))
+		app.SetTxExecutor(STMTxExecutor(app.GetStoreKeys(), workers, preEstimate, app.EvmKeeper, txConfig.TxDecoder()))
+	case "", srvconfig.BlockExecutorSequential:
+		app.SetTxExecutor(DefaultTxExecutor)
+	default:
+		panic(fmt.Errorf("unknown EVM block executor: %s", executor))
+	}
+
 	return app
 }
 

app/executor.go

@@ -5,39 +5,64 @@ import (
 	"io"
 	"sync/atomic"
 
+	"cosmossdk.io/collections"
+	"cosmossdk.io/log"
 	"cosmossdk.io/store/cachemulti"
 	storetypes "cosmossdk.io/store/types"
 	abci "github.com/cometbft/cometbft/abci/types"
+	cmtproto "github.com/cometbft/cometbft/proto/tendermint/types"
+	authtypes "github.com/cosmos/cosmos-sdk/x/auth/types"
+	banktypes "github.com/cosmos/cosmos-sdk/x/bank/types"
 	evmtypes "github.com/evmos/ethermint/x/evm/types"
 
 	"github.com/cosmos/cosmos-sdk/baseapp"
+	sdk "github.com/cosmos/cosmos-sdk/types"
 
 	blockstm "github.com/crypto-org-chain/go-block-stm"
 )
 
 func DefaultTxExecutor(_ context.Context,
-	blockSize int,
+	txs [][]byte,
 	ms storetypes.MultiStore,
-	deliverTxWithMultiStore func(int, storetypes.MultiStore, map[string]any) *abci.ExecTxResult,
+	deliverTxWithMultiStore func(int, sdk.Tx, storetypes.MultiStore, map[string]any) *abci.ExecTxResult,
 ) ([]*abci.ExecTxResult, error) {
+	blockSize := len(txs)
 	results := make([]*abci.ExecTxResult, blockSize)
 	for i := 0; i < blockSize; i++ {
-		results[i] = deliverTxWithMultiStore(i, ms, nil)
+		results[i] = deliverTxWithMultiStore(i, nil, ms, nil)
 	}
 	return evmtypes.PatchTxResponses(results), nil
 }
 
-func STMTxExecutor(stores []storetypes.StoreKey, workers int) baseapp.TxExecutor {
+type evmKeeper interface {
+	GetParams(ctx sdk.Context) evmtypes.Params
+}
+
+func STMTxExecutor(
+	stores []storetypes.StoreKey,
+	workers int,
+	estimate bool,
+	evmKeeper evmKeeper,
+	txDecoder sdk.TxDecoder,
+) baseapp.TxExecutor {
+	var authStore, bankStore int
 	index := make(map[storetypes.StoreKey]int, len(stores))
 	for i, k := range stores {
+		switch k.Name() {
+		case authtypes.StoreKey:
+			authStore = i
+		case banktypes.StoreKey:
+			bankStore = i
+		}
 		index[k] = i
 	}
 	return func(
 		ctx context.Context,
-		blockSize int,
+		txs [][]byte,
 		ms storetypes.MultiStore,
-		deliverTxWithMultiStore func(int, storetypes.MultiStore, map[string]any) *abci.ExecTxResult,
+		deliverTxWithMultiStore func(int, sdk.Tx, storetypes.MultiStore, map[string]any) *abci.ExecTxResult,
 	) ([]*abci.ExecTxResult, error) {
+		blockSize := len(txs)
 		if blockSize == 0 {
 			return nil, nil
 		}
@@ -47,12 +72,27 @@ func STMTxExecutor(stores []storetypes.StoreKey, workers int) baseapp.TxExecutor
 			m := make(map[string]any)
 			incarnationCache[i].Store(&m)
 		}
-		if err := blockstm.ExecuteBlock(
+
+		var estimates map[int]blockstm.MultiLocations
+		memTxs := make([]sdk.Tx, len(txs))
+		if estimate {
+			for i, rawTx := range txs {
+				if memTx, err := txDecoder(rawTx); err == nil {
+					memTxs[i] = memTx
+				}
+			}
+			// pre-estimation
+			evmDenom := evmKeeper.GetParams(sdk.NewContext(ms, cmtproto.Header{}, false, log.NewNopLogger())).EvmDenom
+			estimates = preEstimates(memTxs, authStore, bankStore, evmDenom)
+		}
+
+		if err := blockstm.ExecuteBlockWithEstimates(
 			ctx,
 			blockSize,
 			index,
 			stmMultiStoreWrapper{ms},
 			workers,
+			estimates,
 			func(txn blockstm.TxnIndex, ms blockstm.MultiStore) {
 				var cache map[string]any
 
@@ -63,8 +103,7 @@ func STMTxExecutor(stores []storetypes.StoreKey, workers int) baseapp.TxExecutor
 					cache = *v
 				}
 
-				result := deliverTxWithMultiStore(int(txn), msWrapper{ms}, cache)
-				results[txn] = result
+				results[txn] = deliverTxWithMultiStore(int(txn), memTxs[txn], msWrapper{ms}, cache)
 
 				if v != nil {
 					incarnationCache[txn].Store(v)
@@ -146,3 +185,43 @@ func (ms stmMultiStoreWrapper) GetKVStore(key storetypes.StoreKey) storetypes.KV
 func (ms stmMultiStoreWrapper) GetObjKVStore(key storetypes.StoreKey) storetypes.ObjKVStore {
 	return ms.MultiStore.GetObjKVStore(key)
 }
+
+// preEstimates returns a static estimation of the written keys for each transaction.
+// NOTE: make sure it sync with the latest sdk logic when sdk upgrade.
+func preEstimates(txs []sdk.Tx, authStore, bankStore int, evmDenom string) map[int]blockstm.MultiLocations {
+	estimates := make(map[int]blockstm.MultiLocations, len(txs))
+	for i, tx := range txs {
+		feeTx, ok := tx.(sdk.FeeTx)
+		if !ok {
+			continue
+		}
+		feePayer := sdk.AccAddress(feeTx.FeePayer())
+
+		// account key
+		accKey, err := collections.EncodeKeyWithPrefix(
+			authtypes.AddressStoreKeyPrefix,
+			sdk.AccAddressKey,
+			feePayer,
+		)
+		if err != nil {
+			continue
+		}
+
+		// balance key
+		balanceKey, err := collections.EncodeKeyWithPrefix(
+			banktypes.BalancesPrefix,
+			collections.PairKeyCodec(sdk.AccAddressKey, collections.StringKey),
+			collections.Join(feePayer, evmDenom),
+		)
+		if err != nil {
+			continue
+		}
+
+		estimates[i] = blockstm.MultiLocations{
+			authStore: {accKey},
+			bankStore: {balanceKey},
+		}
+	}
+
+	return estimates
+}

go.mod

@@ -7,6 +7,7 @@ toolchain go1.21.5
 require (
 	cosmossdk.io/api v0.7.5
 	cosmossdk.io/client/v2 v2.0.0-beta.1
+	cosmossdk.io/collections v0.4.0
 	cosmossdk.io/core v0.11.0
 	cosmossdk.io/errors v1.0.1
 	cosmossdk.io/log v1.3.1
@@ -28,7 +29,7 @@ require (
 	github.com/cosmos/ibc-go/modules/capability v1.0.0
 	github.com/cosmos/ibc-go/v8 v8.1.0
 	github.com/cosmos/rosetta v0.50.3-1
-	github.com/crypto-org-chain/go-block-stm v0.0.0-20240806075927-09a64748f883
+	github.com/crypto-org-chain/go-block-stm v0.0.0-20240911081142-92839e79a3ae
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/ethereum/go-ethereum v1.10.26
 	github.com/gogo/protobuf v1.3.2
@@ -65,7 +66,6 @@ require (
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.1.6 // indirect
 	cloud.google.com/go/storage v1.36.0 // indirect
-	cosmossdk.io/collections v0.4.0 // indirect
 	cosmossdk.io/depinject v1.0.0-alpha.4 // indirect
 	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
@@ -243,10 +243,10 @@ require (
 
 replace (
 	// release/v0.50.x
-	cosmossdk.io/client/v2 => github.com/crypto-org-chain/cosmos-sdk/client/v2 v2.0.0-20240722033504-50f1fa0c49d1
-	cosmossdk.io/store => github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240722033504-50f1fa0c49d1
-	cosmossdk.io/x/tx => github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240722033504-50f1fa0c49d1
-	github.com/cosmos/cosmos-sdk => github.com/crypto-org-chain/cosmos-sdk v0.50.6-0.20240722033504-50f1fa0c49d1
+	cosmossdk.io/client/v2 => github.com/crypto-org-chain/cosmos-sdk/client/v2 v2.0.0-20240911084450-6870ba130be2
+	cosmossdk.io/store => github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240911084450-6870ba130be2
+	cosmossdk.io/x/tx => github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240911084450-6870ba130be2
+	github.com/cosmos/cosmos-sdk => github.com/crypto-org-chain/cosmos-sdk v0.50.6-0.20240911084450-6870ba130be2
 )
 
 replace (

go.sum

@@ -411,16 +411,16 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/crypto-org-chain/btree v0.0.0-20240406140148-2687063b042c h1:MOgfS4+FBB8cMkDE2j2VBVsbY+HCkPIu0YsJ/9bbGeQ=
 github.com/crypto-org-chain/btree v0.0.0-20240406140148-2687063b042c/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
-github.com/crypto-org-chain/cosmos-sdk v0.50.6-0.20240722033504-50f1fa0c49d1 h1:xahHemSiT79xgh8Ig8zOTeHSLHt9FfPzViK7rATWhUM=
-github.com/crypto-org-chain/cosmos-sdk v0.50.6-0.20240722033504-50f1fa0c49d1/go.mod h1:Rb43DdB0i/rKcCN69Tg2X3+zA4WhJ7MC8K3a6Ezh38E=
-github.com/crypto-org-chain/cosmos-sdk/client/v2 v2.0.0-20240722033504-50f1fa0c49d1 h1:zqTYZqMKnv15UkKwCEbGqshZ6tWczctBYP53FxmEiF8=
-github.com/crypto-org-chain/cosmos-sdk/client/v2 v2.0.0-20240722033504-50f1fa0c49d1/go.mod h1:W5sR4asmVDUhJpEmuXTUBkk/yEefKlXTjVWcNciVSR0=
-github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240722033504-50f1fa0c49d1 h1:ZlezTiQu9pYpVO+6sB9+W3fvthIpV1GgSI8kPjw+v5s=
-github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240722033504-50f1fa0c49d1/go.mod h1:gjE3DZe4t/+VeIk6CmrouyqiuDbZ7QOVDDq3nLqBTpg=
-github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240722033504-50f1fa0c49d1 h1:r0ALP31Wnw19FqEmqzsK2SFNqdMetHshnM/X/FeJRIo=
-github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240722033504-50f1fa0c49d1/go.mod h1:RTiTs4hkXG6IvYGknvB8p79YgjYJdcbzLUOGJChsPnY=
-github.com/crypto-org-chain/go-block-stm v0.0.0-20240806075927-09a64748f883 h1:Oj7VvlK8iXRaugnpGA8CBXGrgkyigji+Ae5weSRD85I=
-github.com/crypto-org-chain/go-block-stm v0.0.0-20240806075927-09a64748f883/go.mod h1:iwQTX9xMX8NV9k3o2BiWXA0SswpsZrDk5q3gA7nWYiE=
+github.com/crypto-org-chain/cosmos-sdk v0.50.6-0.20240911084450-6870ba130be2 h1:4SoAvnxDaiIWcgm6XOmPDIdCf4/WNhNYLXGbij1eaA0=
+github.com/crypto-org-chain/cosmos-sdk v0.50.6-0.20240911084450-6870ba130be2/go.mod h1:Rb43DdB0i/rKcCN69Tg2X3+zA4WhJ7MC8K3a6Ezh38E=
+github.com/crypto-org-chain/cosmos-sdk/client/v2 v2.0.0-20240911084450-6870ba130be2 h1:5oGZtOUcauk9dtv+8BCfj2PEQyXEEEV+K3sP4OSvBmg=
+github.com/crypto-org-chain/cosmos-sdk/client/v2 v2.0.0-20240911084450-6870ba130be2/go.mod h1:W5sR4asmVDUhJpEmuXTUBkk/yEefKlXTjVWcNciVSR0=
+github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240911084450-6870ba130be2 h1:CGh5I0L6IYhe0AJevb4vf5TE3ru+qAgMs437BlWCwo8=
+github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240911084450-6870ba130be2/go.mod h1:gjE3DZe4t/+VeIk6CmrouyqiuDbZ7QOVDDq3nLqBTpg=
+github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240911084450-6870ba130be2 h1:mxlOSCru7YgmX055rrlkCSUu0D8lAqJ8Dnhp0yXCBuM=
+github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240911084450-6870ba130be2/go.mod h1:RTiTs4hkXG6IvYGknvB8p79YgjYJdcbzLUOGJChsPnY=
+github.com/crypto-org-chain/go-block-stm v0.0.0-20240911081142-92839e79a3ae h1:gakWYsVubWX8P9NpxaPnvg0UJYfIZigfko5WN57t7OA=
+github.com/crypto-org-chain/go-block-stm v0.0.0-20240911081142-92839e79a3ae/go.mod h1:iwQTX9xMX8NV9k3o2BiWXA0SswpsZrDk5q3gA7nWYiE=
 github.com/crypto-org-chain/go-ethereum v1.10.20-0.20240425065928-ebb09502e7a7 h1:V43F3JFcqG4MUThf9W/DytnPblpR6CcaLBw2Wx6zTgE=
 github.com/crypto-org-chain/go-ethereum v1.10.20-0.20240425065928-ebb09502e7a7/go.mod h1:+a8pUj1tOyJ2RinsNQD4326YS+leSoKGiG/uVVb0x6Y=
 github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE=

gomod2nix.toml

@@ -20,7 +20,7 @@ schema = 3
     version = "v0.7.5"
     hash = "sha256-Nuw697sJr56kU3EU7DV1eYNkyI76psznIVqYAV6RfbQ="
   [mod."cosmossdk.io/client/v2"]
-    version = "v2.0.0-20240722033504-50f1fa0c49d1"
+    version = "v2.0.0-20240911084450-6870ba130be2"
     hash = "sha256-60hmufv3Ml4Pv3zNwgn8eeqlEINOR6n9MKr2QHddoxo="
     replaced = "github.com/crypto-org-chain/cosmos-sdk/client/v2"
   [mod."cosmossdk.io/collections"]
@@ -42,7 +42,7 @@ schema = 3
     version = "v1.3.0"
     hash = "sha256-EEFK43Cr0g0ndhQhkIKher0FqV3mvkmE9z0sP7uVSHg="
   [mod."cosmossdk.io/store"]
-    version = "v0.0.0-20240722033504-50f1fa0c49d1"
+    version = "v0.0.0-20240911084450-6870ba130be2"
     hash = "sha256-Dm3sSZNJBcnBF33PULoTpK4rkNQbsZl0DfTqH1GPCQM="
     replaced = "github.com/crypto-org-chain/cosmos-sdk/store"
   [mod."cosmossdk.io/tools/confix"]
@@ -55,7 +55,7 @@ schema = 3
     version = "v0.1.0"
     hash = "sha256-/gWvrqvy6bW90+NU66T+9QysYgvG1VbwfYJZ8tkqpeA="
   [mod."cosmossdk.io/x/tx"]
-    version = "v0.0.0-20240722033504-50f1fa0c49d1"
+    version = "v0.0.0-20240911084450-6870ba130be2"
     hash = "sha256-xT5IdapEx1h46ofBpxcBQfzGF2EntmC8xZl7aym/6xE="
     replaced = "github.com/crypto-org-chain/cosmos-sdk/x/tx"
   [mod."cosmossdk.io/x/upgrade"]
@@ -163,8 +163,8 @@ schema = 3
     version = "v1.0.0-beta.5"
     hash = "sha256-Fy/PbsOsd6iq0Njy3DVWK6HqWsogI+MkE8QslHGWyVg="
   [mod."github.com/cosmos/cosmos-sdk"]
-    version = "v0.50.6-0.20240722033504-50f1fa0c49d1"
-    hash = "sha256-OjJgi6tq5c4czeQLZVeJRBo4s4kC37h4IFT/GGvF7G0="
+    version = "v0.50.6-0.20240911084450-6870ba130be2"
+    hash = "sha256-kl2sLe8vITIguRGtdeviDeP3R5JIbRg+eDsheGd4PqM="
     replaced = "github.com/crypto-org-chain/cosmos-sdk"
   [mod."github.com/cosmos/go-bip39"]
     version = "v1.0.0"
@@ -203,8 +203,8 @@ schema = 3
     version = "v0.0.24"
     hash = "sha256-4vUukHONOjNn0qfQr4esK6TWfPWsIp+rbdz65og84lw="
   [mod."github.com/crypto-org-chain/go-block-stm"]
-    version = "v0.0.0-20240806075927-09a64748f883"
-    hash = "sha256-DuqcnTqpUVRxV+I+CaBuDg/1CLmMeXdw4VEFeuCsUbU="
+    version = "v0.0.0-20240911081142-92839e79a3ae"
+    hash = "sha256-8MhSeC5BB5BwOet3k3Rfua0TBeBATtLSwaW6s5WpYCM="
   [mod."github.com/danieljoos/wincred"]
     version = "v1.2.0"
     hash = "sha256-LHcvTJCc8++bFndbd8ZgMSTe4L5h2C4rN+cSWHCz54Y="

server/config/config.go

@@ -134,6 +134,8 @@ type EVMConfig struct {
 	BlockExecutor string `mapstructure:"block-executor"`
 	// BlockSTMWorkers is the number of workers for block-stm execution, `0` means using all available CPUs.
 	BlockSTMWorkers int `mapstructure:"block-stm-workers"`
+	// BlockSTMPreEstimate is the flag to enable pre-estimation for block-stm execution.
+	BlockSTMPreEstimate bool `mapstructure:"block-stm-pre-estimate"`
 }
 
 // JSONRPCConfig defines configuration for the EVM RPC server.
@@ -407,10 +409,11 @@ func GetConfig(v *viper.Viper) (Config, error) {
 	return Config{
 		Config: cfg,
 		EVM: EVMConfig{
-			Tracer:          v.GetString("evm.tracer"),
-			MaxTxGasWanted:  v.GetUint64("evm.max-tx-gas-wanted"),
-			BlockExecutor:   v.GetString("evm.block-executor"),
-			BlockSTMWorkers: v.GetInt("evm.block-stm-workers"),
+			Tracer:              v.GetString("evm.tracer"),
+			MaxTxGasWanted:      v.GetUint64("evm.max-tx-gas-wanted"),
+			BlockExecutor:       v.GetString("evm.block-executor"),
+			BlockSTMWorkers:     v.GetInt("evm.block-stm-workers"),
+			BlockSTMPreEstimate: v.GetBool("evm.block-stm-pre-estimate"),
 		},
 		JSONRPC: JSONRPCConfig{
 			Enable:                   v.GetBool("json-rpc.enable"),

server/config/toml.go

@@ -35,6 +35,8 @@ max-tx-gas-wanted = {{ .EVM.MaxTxGasWanted }}
 block-executor = "{{ .EVM.BlockExecutor }}"
 # BlockSTMWorkers is the number of workers for block-stm execution, 0 means using all available CPUs.
 block-stm-workers = {{ .EVM.BlockSTMWorkers }}
+# BlockSTMPreEstimate is the flag to enable pre-estimation for block-stm execution.
+block-stm-pre-estimate = {{ .EVM.BlockSTMPreEstimate }}
 
 ###############################################################################
 ###                           JSON RPC Configuration                        ###

server/flags/flags.go

@@ -76,10 +76,11 @@ const (
 
 // EVM flags
 const (
-	EVMTracer          = "evm.tracer"
-	EVMMaxTxGasWanted  = "evm.max-tx-gas-wanted"
-	EVMBlockExecutor   = "evm.block-executor"
-	EVMBlockSTMWorkers = "evm.block-stm-workers"
+	EVMTracer              = "evm.tracer"
+	EVMMaxTxGasWanted      = "evm.max-tx-gas-wanted"
+	EVMBlockExecutor       = "evm.block-executor"
+	EVMBlockSTMWorkers     = "evm.block-stm-workers"
+	EVMBlockSTMPreEstimate = "evm.block-stm-pre-estimate"
 )
 
 // TLS flags