From e841494dd3dadcb4caea4f0c89f53efe1c523b30 Mon Sep 17 00:00:00 2001 From: jules01 Date: Wed, 15 Nov 2023 12:26:36 +0200 Subject: [PATCH 1/3] - integrated p2p resource limiter config --- cmd/node/config/p2p.toml | 4 ++++ cmd/seednode/config/p2p.toml | 4 ++++ config/tomlConfig_test.go | 9 +++++++++ go.mod | 6 +++--- go.sum | 4 ++-- .../p2p/networkSharding-hbv2/networkSharding_test.go | 3 +++ integrationTests/testInitializer.go | 9 +++++++++ p2p/config/config.go | 3 +++ p2p/constants.go | 3 +++ testscommon/components/components.go | 3 +++ 10 files changed, 43 insertions(+), 5 deletions(-) diff --git a/cmd/node/config/p2p.toml b/cmd/node/config/p2p.toml index b592da9f06f..62d30fd19f7 100644 --- a/cmd/node/config/p2p.toml +++ b/cmd/node/config/p2p.toml @@ -23,6 +23,10 @@ [Node.Transports.TCP] ListenAddress = "/ip4/0.0.0.0/tcp/%d" # TCP listen address PreventPortReuse = false + [Node.ResourceLimiter] + Type = "default autoscale" #available options "default autoscale", "infinite", "default with manual scale". + ManualSystemMemoryInMB = 0 # not taken into account if the type is not "default with manual scale" + ManualMaximumFD = 0 # not taken into account if the type is not "default with manual scale" # P2P peer discovery section diff --git a/cmd/seednode/config/p2p.toml b/cmd/seednode/config/p2p.toml index 3dc337ad7dd..2c1a92717c9 100644 --- a/cmd/seednode/config/p2p.toml +++ b/cmd/seednode/config/p2p.toml @@ -22,6 +22,10 @@ [Node.Transports.TCP] ListenAddress = "/ip4/0.0.0.0/tcp/%d" # TCP listen address PreventPortReuse = true # seeder nodes will need to enable this option + [Node.ResourceLimiter] + Type = "default with manual scale" + ManualSystemMemoryInMB = 65536 # pretend that the host running the seeder has more RAM so it can handle more connections + ManualMaximumFD = 1048576 # P2P peer discovery section diff --git a/config/tomlConfig_test.go b/config/tomlConfig_test.go index a844be408c0..cdf90eae957 100644 --- a/config/tomlConfig_test.go +++ b/config/tomlConfig_test.go @@ -493,6 +493,10 @@ func TestP2pConfig(t *testing.T) { [Node.Transports.TCP] ListenAddress = "/ip4/0.0.0.0/tcp/%d" PreventPortReuse = true + [Node.ResourceLimiter] + Type = "default autoscale" #available options "default autoscale", "infinite", "default with manual scale". + ManualSystemMemoryInMB = 1 # not taken into account if the type is not "default with manual scale" + ManualMaximumFD = 2 # not taken into account if the type is not "default with manual scale" [KadDhtPeerDiscovery] Enabled = false @@ -529,6 +533,11 @@ func TestP2pConfig(t *testing.T) { WebSocketAddress: "/ip4/0.0.0.0/tcp/%d/ws", WebTransportAddress: "/ip4/0.0.0.0/udp/%d/quic-v1/webtransport", }, + ResourceLimiter: p2pConfig.P2PResourceLimiterConfig{ + Type: "default autoscale", + ManualSystemMemoryInMB: 1, + ManualMaximumFD: 2, + }, }, KadDhtPeerDiscovery: p2pConfig.KadDhtPeerDiscoveryConfig{ ProtocolID: protocolID, diff --git a/go.mod b/go.mod index 582316e0e6f..b8ae605ce9c 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/google/gops v0.3.18 github.com/gorilla/websocket v1.5.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/multiversx/mx-chain-communication-go v1.0.8 + github.com/multiversx/mx-chain-communication-go v1.0.9-0.20231115091649-90797a86f941 github.com/multiversx/mx-chain-core-go v1.2.18 github.com/multiversx/mx-chain-crypto-go v1.2.9 github.com/multiversx/mx-chain-es-indexer-go v1.4.13 @@ -25,12 +25,14 @@ require ( github.com/multiversx/mx-chain-vm-v1_2-go v1.2.61 github.com/multiversx/mx-chain-vm-v1_3-go v1.3.62 github.com/multiversx/mx-chain-vm-v1_4-go v1.4.90 + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pelletier/go-toml v1.9.3 github.com/pkg/errors v0.9.1 github.com/shirou/gopsutil v3.21.11+incompatible github.com/stretchr/testify v1.8.4 github.com/urfave/cli v1.22.10 golang.org/x/crypto v0.10.0 + golang.org/x/sys v0.10.0 gopkg.in/go-playground/validator.v8 v8.18.2 ) @@ -136,7 +138,6 @@ require ( github.com/onsi/ginkgo/v2 v2.9.7 // indirect github.com/opencontainers/runtime-spec v1.0.2 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect @@ -178,7 +179,6 @@ require ( golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.11.0 // indirect golang.org/x/sync v0.2.0 // indirect - golang.org/x/sys v0.10.0 // indirect golang.org/x/text v0.10.0 // indirect golang.org/x/tools v0.9.1 // indirect gonum.org/v1/gonum v0.11.0 // indirect diff --git a/go.sum b/go.sum index 273ffbe6650..3f2175d70d5 100644 --- a/go.sum +++ b/go.sum @@ -384,8 +384,8 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUYwbO0993uPI= github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= -github.com/multiversx/mx-chain-communication-go v1.0.8 h1:sTx4Vmx+QCpngUFq/LF/Ka8bevlK2vMxfclE284twfc= -github.com/multiversx/mx-chain-communication-go v1.0.8/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= +github.com/multiversx/mx-chain-communication-go v1.0.9-0.20231115091649-90797a86f941 h1:qKRGz9KiLg1dX2zGVjx62iUdwvpow74yAPKLZRaUEa8= +github.com/multiversx/mx-chain-communication-go v1.0.9-0.20231115091649-90797a86f941/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= github.com/multiversx/mx-chain-core-go v1.2.18 h1:fnub2eFL7XYOLrKKVZAPPsaM1TWEnaK5qqY3FLUv168= github.com/multiversx/mx-chain-core-go v1.2.18/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= github.com/multiversx/mx-chain-crypto-go v1.2.9 h1:OEfF2kOQrtzUl273Z3DEcshjlTVUfPpJMd0R0SvTrlU= diff --git a/integrationTests/p2p/networkSharding-hbv2/networkSharding_test.go b/integrationTests/p2p/networkSharding-hbv2/networkSharding_test.go index a99a9cf392b..c11c73838c5 100644 --- a/integrationTests/p2p/networkSharding-hbv2/networkSharding_test.go +++ b/integrationTests/p2p/networkSharding-hbv2/networkSharding_test.go @@ -22,6 +22,9 @@ func createDefaultConfig() p2pConfig.P2PConfig { ListenAddress: p2p.LocalHostListenAddrWithIp4AndTcp, }, }, + ResourceLimiter: p2pConfig.P2PResourceLimiterConfig{ + Type: p2p.DefaultWithScaleResourceLimiter, + }, }, KadDhtPeerDiscovery: p2pConfig.KadDhtPeerDiscoveryConfig{ Enabled: true, diff --git a/integrationTests/testInitializer.go b/integrationTests/testInitializer.go index 8d08a89f6fe..6a72d118a50 100644 --- a/integrationTests/testInitializer.go +++ b/integrationTests/testInitializer.go @@ -144,6 +144,9 @@ func createP2PConfig(initialPeerList []string) p2pConfig.P2PConfig { ListenAddress: p2p.LocalHostListenAddrWithIp4AndTcp, }, }, + ResourceLimiter: p2pConfig.P2PResourceLimiterConfig{ + Type: p2p.DefaultWithScaleResourceLimiter, + }, }, KadDhtPeerDiscovery: p2pConfig.KadDhtPeerDiscoveryConfig{ Enabled: true, @@ -237,6 +240,9 @@ func CreateP2PConfigWithNoDiscovery() p2pConfig.P2PConfig { ListenAddress: p2p.LocalHostListenAddrWithIp4AndTcp, }, }, + ResourceLimiter: p2pConfig.P2PResourceLimiterConfig{ + Type: p2p.DefaultWithScaleResourceLimiter, + }, }, KadDhtPeerDiscovery: p2pConfig.KadDhtPeerDiscoveryConfig{ Enabled: false, @@ -264,6 +270,9 @@ func CreateMessengerWithNoDiscoveryAndPeersRatingHandler(peersRatingHanlder p2p. ListenAddress: p2p.LocalHostListenAddrWithIp4AndTcp, }, }, + ResourceLimiter: p2pConfig.P2PResourceLimiterConfig{ + Type: p2p.DefaultWithScaleResourceLimiter, + }, }, KadDhtPeerDiscovery: p2pConfig.KadDhtPeerDiscoveryConfig{ Enabled: false, diff --git a/p2p/config/config.go b/p2p/config/config.go index aa332b6b898..fb64a203949 100644 --- a/p2p/config/config.go +++ b/p2p/config/config.go @@ -11,6 +11,9 @@ type P2PTransportConfig = config.TransportConfig // P2PTCPTransport will hold the P2P TCP transport config type P2PTCPTransport = config.TCPProtocolConfig +// P2PResourceLimiterConfig will hold the P2P resource limiter configs +type P2PResourceLimiterConfig = config.ResourceLimiterConfig + // NodeConfig will hold basic p2p settings type NodeConfig = config.NodeConfig diff --git a/p2p/constants.go b/p2p/constants.go index 33a8df0ba84..8a6db9caeb4 100644 --- a/p2p/constants.go +++ b/p2p/constants.go @@ -25,5 +25,8 @@ const ConnectionWatcherTypePrint = p2p.ConnectionWatcherTypePrint // LocalHostListenAddrWithIp4AndTcp defines the local host listening ip v.4 address and TCP const LocalHostListenAddrWithIp4AndTcp = p2p.LocalHostListenAddrWithIp4AndTcp +// DefaultWithScaleResourceLimiter defines the default & autoscale resource limiter +const DefaultWithScaleResourceLimiter = p2p.DefaultWithScaleResourceLimiter + // BroadcastMethod defines the broadcast method of the message type BroadcastMethod = p2p.BroadcastMethod diff --git a/testscommon/components/components.go b/testscommon/components/components.go index 2aec9cc4277..cc4ec1b03ab 100644 --- a/testscommon/components/components.go +++ b/testscommon/components/components.go @@ -249,6 +249,9 @@ func GetNetworkFactoryArgs() networkComp.NetworkComponentsFactoryArgs { ListenAddress: p2p.LocalHostListenAddrWithIp4AndTcp, }, }, + ResourceLimiter: p2pConfig.P2PResourceLimiterConfig{ + Type: p2p.DefaultWithScaleResourceLimiter, + }, }, KadDhtPeerDiscovery: p2pConfig.KadDhtPeerDiscoveryConfig{ Enabled: false, From 8e039101da3fe2836770e1d7110e2e077541e1a7 Mon Sep 17 00:00:00 2001 From: jules01 Date: Wed, 15 Nov 2023 12:52:50 +0200 Subject: [PATCH 2/3] - added definition in the fullArchiveP2P.toml file --- cmd/node/config/fullArchiveP2P.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/node/config/fullArchiveP2P.toml b/cmd/node/config/fullArchiveP2P.toml index 524687a3c50..0dd790a83f6 100644 --- a/cmd/node/config/fullArchiveP2P.toml +++ b/cmd/node/config/fullArchiveP2P.toml @@ -23,6 +23,10 @@ [Node.Transports.TCP] ListenAddress = "/ip4/0.0.0.0/tcp/%d" # TCP listen address PreventPortReuse = false + [Node.ResourceLimiter] + Type = "default autoscale" #available options "default autoscale", "infinite", "default with manual scale". + ManualSystemMemoryInMB = 0 # not taken into account if the type is not "default with manual scale" + ManualMaximumFD = 0 # not taken into account if the type is not "default with manual scale" # P2P peer discovery section From 3cbe7751867878f9a27484406d9eece8fb4a460b Mon Sep 17 00:00:00 2001 From: jules01 Date: Thu, 16 Nov 2023 09:58:07 +0200 Subject: [PATCH 3/3] - proper version --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b8ae605ce9c..af98109070b 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/google/gops v0.3.18 github.com/gorilla/websocket v1.5.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/multiversx/mx-chain-communication-go v1.0.9-0.20231115091649-90797a86f941 + github.com/multiversx/mx-chain-communication-go v1.0.9 github.com/multiversx/mx-chain-core-go v1.2.18 github.com/multiversx/mx-chain-crypto-go v1.2.9 github.com/multiversx/mx-chain-es-indexer-go v1.4.13 diff --git a/go.sum b/go.sum index 3f2175d70d5..018b147dcce 100644 --- a/go.sum +++ b/go.sum @@ -384,8 +384,8 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUYwbO0993uPI= github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= -github.com/multiversx/mx-chain-communication-go v1.0.9-0.20231115091649-90797a86f941 h1:qKRGz9KiLg1dX2zGVjx62iUdwvpow74yAPKLZRaUEa8= -github.com/multiversx/mx-chain-communication-go v1.0.9-0.20231115091649-90797a86f941/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= +github.com/multiversx/mx-chain-communication-go v1.0.9 h1:unntj9uANJVzANQ4qaE+yth3Y2Tuih6c9ukKAaHOECs= +github.com/multiversx/mx-chain-communication-go v1.0.9/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4= github.com/multiversx/mx-chain-core-go v1.2.18 h1:fnub2eFL7XYOLrKKVZAPPsaM1TWEnaK5qqY3FLUv168= github.com/multiversx/mx-chain-core-go v1.2.18/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84= github.com/multiversx/mx-chain-crypto-go v1.2.9 h1:OEfF2kOQrtzUl273Z3DEcshjlTVUfPpJMd0R0SvTrlU=