From 8aa840dbd33fefc8e21b691de7a090ddf2d5c6f3 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 12 Aug 2024 10:47:31 +0100 Subject: [PATCH 01/55] initial quic prototype --- consensus/reactor.go | 10 +- go.mod | 31 +++--- go.sum | 64 ++++++----- libs/net/net.go | 44 +++++++- node/node.go | 3 +- p2p/conn/connection.go | 34 +++--- p2p/conn_set.go | 15 +-- p2p/errors.go | 3 +- p2p/netaddress.go | 61 ++++++++++- p2p/peer.go | 199 ++++++++------------------------- p2p/pex/pex_reactor.go | 6 +- p2p/test_util.go | 2 + p2p/transport.go | 244 ++++++++++------------------------------- rpc/core/net.go | 8 +- 14 files changed, 289 insertions(+), 435 deletions(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index eec5afa8d0..5649aa1129 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -350,7 +350,7 @@ func (conR *Reactor) ReceiveEnvelope(e p2p.Envelope) { if votes := ourVotes.ToProto(); votes != nil { eMsg.Votes = *votes } - if p2p.TrySendEnvelopeShim(e.Src, p2p.Envelope{ //nolint: staticcheck + if p2p.SendEnvelopeShim(e.Src, p2p.Envelope{ //nolint: staticcheck ChannelID: VoteSetBitsChannel, Message: eMsg, }, conR.Logger) { @@ -1014,7 +1014,7 @@ OUTER_LOOP: if rs.Height == prs.Height { if maj23, ok := rs.Votes.Prevotes(prs.Round).TwoThirdsMajority(); ok { - if p2p.TrySendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck + if p2p.SendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck ChannelID: StateChannel, Message: &cmtcons.VoteSetMaj23{ Height: prs.Height, @@ -1043,7 +1043,7 @@ OUTER_LOOP: prs := ps.GetRoundState() if rs.Height == prs.Height { if maj23, ok := rs.Votes.Precommits(prs.Round).TwoThirdsMajority(); ok { - if p2p.TrySendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck + if p2p.SendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck ChannelID: StateChannel, Message: &cmtcons.VoteSetMaj23{ Height: prs.Height, @@ -1073,7 +1073,7 @@ OUTER_LOOP: if rs.Height == prs.Height && prs.ProposalPOLRound >= 0 { if maj23, ok := rs.Votes.Prevotes(prs.ProposalPOLRound).TwoThirdsMajority(); ok { - if p2p.TrySendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck + if p2p.SendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck ChannelID: StateChannel, Message: &cmtcons.VoteSetMaj23{ Height: prs.Height, @@ -1105,7 +1105,7 @@ OUTER_LOOP: if prs.CatchupCommitRound != -1 && prs.Height > 0 && prs.Height <= conR.conS.blockStore.Height() && prs.Height >= conR.conS.blockStore.Base() { if commit := conR.conS.LoadCommit(prs.Height); commit != nil { - if p2p.TrySendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck + if p2p.SendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck ChannelID: StateChannel, Message: &cmtcons.VoteSetMaj23{ Height: prs.Height, diff --git a/go.mod b/go.mod index 0b770cedf9..f5e94ba7f5 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,8 @@ require ( github.com/minio/highwayhash v1.0.2 github.com/ory/dockertest v3.3.5+incompatible github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/client_golang v1.19.1 + github.com/quic-go/quic-go v0.46.0 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 github.com/rs/cors v1.8.3 github.com/sasha-s/go-deadlock v0.3.1 @@ -50,11 +51,11 @@ require ( go.opentelemetry.io/otel v1.24.0 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.18.0 go.opentelemetry.io/otel/sdk v1.21.0 - golang.org/x/crypto v0.21.0 - golang.org/x/net v0.23.0 + golang.org/x/crypto v0.23.0 + golang.org/x/net v0.25.0 gonum.org/v1/gonum v0.12.0 google.golang.org/grpc v1.60.0 - google.golang.org/protobuf v1.31.0 + google.golang.org/protobuf v1.33.0 ) require ( @@ -129,6 +130,7 @@ require ( github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/go-toolsmith/astcast v1.1.0 // indirect github.com/go-toolsmith/astcopy v1.1.0 // indirect github.com/go-toolsmith/astequal v1.1.0 // indirect @@ -201,7 +203,6 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mbilski/exhaustivestruct v1.2.0 // indirect github.com/mgechev/revive v1.3.1 // indirect github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect @@ -216,6 +217,7 @@ require ( github.com/nishanths/predeclared v0.2.2 // indirect github.com/nunnatsa/ginkgolinter v0.9.0 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/onsi/ginkgo/v2 v2.9.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc2 // indirect github.com/opencontainers/runc v1.1.3 // indirect @@ -226,8 +228,8 @@ require ( github.com/pkg/profile v1.7.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/polyfloyd/go-errorlint v1.4.5 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/quasilyte/go-ruleguard v0.4.0 // indirect github.com/quasilyte/gogrep v0.5.0 // indirect @@ -278,16 +280,17 @@ require ( go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/atomic v1.10.0 // indirect + go.uber.org/mock v0.4.0 // indirect go.uber.org/multierr v1.10.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/exp/typeparams v0.0.0-20230307190834-24139beb5833 // indirect - golang.org/x/mod v0.12.0 // indirect - golang.org/x/sync v0.5.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.13.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + golang.org/x/tools v0.21.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index 1933375e28..76ec8fc0e3 100644 --- a/go.sum +++ b/go.sum @@ -319,6 +319,8 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-toolsmith/astcast v1.1.0 h1:+JN9xZV1A+Re+95pgnMgDboWNVnIMMQXwfBwLRPgSC8= github.com/go-toolsmith/astcast v1.1.0/go.mod h1:qdcuFWeGGS2xX5bLM/c3U9lewg7+Zu4mr+xPwZIB4ZU= github.com/go-toolsmith/astcopy v1.1.0 h1:YGwBN0WM+ekI/6SS6+52zLDEf8Yvp3n2seZITCUBt5s= @@ -616,8 +618,6 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mbilski/exhaustivestruct v1.2.0 h1:wCBmUnSYufAHO6J4AVWY6ff+oxWxsVFrwgOdMUQePUo= github.com/mbilski/exhaustivestruct v1.2.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc= github.com/mgechev/revive v1.3.1 h1:OlQkcH40IB2cGuprTPcjB0iIUddgVZgGmDX3IAMR8D4= @@ -667,8 +667,8 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo/v2 v2.8.0 h1:pAM+oBNPrpXRs+E/8spkeGx9QgekbRVyr74EUvRVOUI= -github.com/onsi/ginkgo/v2 v2.8.0/go.mod h1:6JsQiECmxCa3V5st74AL/AmsV482EDdVrGaVW6z3oYU= +github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= +github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -720,20 +720,20 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -749,6 +749,8 @@ github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 h1:TCg2WBOl github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= +github.com/quic-go/quic-go v0.46.0 h1:uuwLClEEyk1DNvchH8uCByQVjo3yKL9opKulExNDs7Y= +github.com/quic-go/quic-go v0.46.0/go.mod h1:1dLehS7TIR64+vxGR70GDcatWTOtMX2PUtnKsjbTurI= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -946,6 +948,8 @@ go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= @@ -965,8 +969,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -977,8 +981,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20230307190834-24139beb5833 h1:jWGQJV4niP+CCmFW9ekjA9Zx8vYORzOUH2/Nl5WPuLQ= @@ -1011,8 +1015,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1060,8 +1064,8 @@ golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1081,8 +1085,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1160,8 +1164,8 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1170,8 +1174,8 @@ golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1184,8 +1188,8 @@ golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1258,8 +1262,8 @@ golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= +golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1346,8 +1350,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/libs/net/net.go b/libs/net/net.go index fa85256fa8..f27e8a3214 100644 --- a/libs/net/net.go +++ b/libs/net/net.go @@ -1,6 +1,10 @@ package net import ( + "context" + "crypto/tls" + "fmt" + "github.com/quic-go/quic-go" "net" "strings" ) @@ -13,14 +17,46 @@ func Connect(protoAddr string) (net.Conn, error) { return conn, err } +// Connect dials the given address and returns a net.Conn. The protoAddr argument should be prefixed with the protocol, +// eg. "tcp://127.0.0.1:8080" or "unix:///tmp/test.sock" +func QuicConnect(protoAddr string) (quic.Connection, error) { + proto, address := ProtocolAndAddress(protoAddr) + tlsConfig := tls.Config{ + MinVersion: tls.VersionTLS12, + } + // TODO better config + quickConfig := quic.Config{ + GetConfigForClient: nil, + Versions: nil, + HandshakeIdleTimeout: 0, + MaxIdleTimeout: 0, + TokenStore: nil, + InitialStreamReceiveWindow: 0, + MaxStreamReceiveWindow: 0, + InitialConnectionReceiveWindow: 0, + MaxConnectionReceiveWindow: 0, + AllowConnectionWindowIncrease: nil, + MaxIncomingStreams: 0, + MaxIncomingUniStreams: 0, + KeepAlivePeriod: 0, + InitialPacketSize: 0, + DisablePathMTUDiscovery: false, + Allow0RTT: false, + EnableDatagrams: false, + Tracer: nil, + } + conn, err := quic.DialAddr(context.Background(), fmt.Sprintf("%s:%s", proto, address), &tlsConfig, &quickConfig) + return conn, err +} + // ProtocolAndAddress splits an address into the protocol and address components. // For instance, "tcp://127.0.0.1:8080" will be split into "tcp" and "127.0.0.1:8080". // If the address has no protocol prefix, the default is "tcp". func ProtocolAndAddress(listenAddr string) (string, string) { - protocol, address := "tcp", listenAddr + protocol, address := "udp", listenAddr parts := strings.SplitN(address, "://", 2) if len(parts) == 2 { - protocol, address = parts[0], parts[1] + address = parts[1] } return protocol, address } @@ -29,12 +65,12 @@ func ProtocolAndAddress(listenAddr string) (string, string) { // Ripped from https://github.com/phayes/freeport. // BSD-licensed. func GetFreePort() (int, error) { - addr, err := net.ResolveTCPAddr("tcp", "localhost:0") + addr, err := net.ResolveTCPAddr("udp", "localhost:0") if err != nil { return 0, err } - l, err := net.ListenTCP("tcp", addr) + l, err := net.ListenTCP("udp", addr) if err != nil { return 0, err } diff --git a/node/node.go b/node/node.go index e2c749fb1f..24b6cdd3a0 100644 --- a/node/node.go +++ b/node/node.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "github.com/quic-go/quic-go" "net" "net/http" "strings" @@ -571,7 +572,7 @@ func createTransport( connFilters = append( connFilters, // ABCI query for address filtering. - func(_ p2p.ConnSet, c net.Conn, _ []net.IP) error { + func(_ p2p.ConnSet, c quic.Connection, _ []net.IP) error { res, err := proxyApp.Query().QuerySync(abci.RequestQuery{ Path: fmt.Sprintf("/p2p/filter/addr/%s", c.RemoteAddr().String()), }) diff --git a/p2p/conn/connection.go b/p2p/conn/connection.go index 87ab2ed28a..b09e29be5b 100644 --- a/p2p/conn/connection.go +++ b/p2p/conn/connection.go @@ -426,7 +426,7 @@ func (c *MConnection) sendRoutine() { FOR_LOOP: for { - var _n int + //var _n int var err error SELECTION: select { @@ -440,12 +440,12 @@ FOR_LOOP: } case <-c.pingTimer.C: c.Logger.Debug("Send Ping") - _n, err = protoWriter.WriteMsg(mustWrapPacket(&tmp2p.PacketPing{})) + _, err = protoWriter.WriteMsg(mustWrapPacket(&tmp2p.PacketPing{})) if err != nil { c.Logger.Error("Failed to send PacketPing", "err", err) break SELECTION } - c.sendMonitor.Update(_n) + //c.sendMonitor.Update(_n) c.Logger.Debug("Starting pong timer", "dur", c.config.PongTimeout) c.pongTimer = time.AfterFunc(c.config.PongTimeout, func() { select { @@ -463,12 +463,12 @@ FOR_LOOP: } case <-c.pong: c.Logger.Debug("Send Pong") - _n, err = protoWriter.WriteMsg(mustWrapPacket(&tmp2p.PacketPong{})) + _, err = protoWriter.WriteMsg(mustWrapPacket(&tmp2p.PacketPong{})) if err != nil { c.Logger.Error("Failed to send PacketPong", "err", err) break SELECTION } - c.sendMonitor.Update(_n) + //c.sendMonitor.Update(_n) c.flush() case <-c.quitSendRoutine: break FOR_LOOP @@ -505,7 +505,7 @@ func (c *MConnection) sendSomePacketMsgs() bool { // Block until .sendMonitor says we can write. // Once we're ready we send more than we asked for, // but amortized it should even out. - c.sendMonitor.Limit(c._maxPacketMsgSize, atomic.LoadInt64(&c.config.SendRate), true) + //c.sendMonitor.Limit(c._maxPacketMsgSize, atomic.LoadInt64(&c.config.SendRate), true) // Now send some PacketMsgs. for i := 0; i < numBatchPacketMsgs; i++ { @@ -542,13 +542,13 @@ func (c *MConnection) sendPacketMsg() bool { // c.Logger.Info("Found a msgPacket to send") // Make & send a PacketMsg from this channel - _n, err := leastChannel.writePacketMsgTo(c.bufConnWriter) + _, err := leastChannel.writePacketMsgTo(c.bufConnWriter) if err != nil { c.Logger.Error("Failed to write PacketMsg", "err", err) c.stopForError(err) return true } - c.sendMonitor.Update(_n) + //c.sendMonitor.Update(_n) c.flushTimer.Set() return false } @@ -565,7 +565,7 @@ func (c *MConnection) recvRoutine() { FOR_LOOP: for { // Block until .recvMonitor says we can read. - c.recvMonitor.Limit(c._maxPacketMsgSize, atomic.LoadInt64(&c.config.RecvRate), true) + //c.recvMonitor.Limit(c._maxPacketMsgSize, atomic.LoadInt64(&c.config.RecvRate), true) // Peek into bufConnReader for debugging /* @@ -584,8 +584,8 @@ FOR_LOOP: // Read packet type var packet tmp2p.Packet - _n, err := protoReader.ReadMsg(&packet) - c.recvMonitor.Update(_n) + _, err := protoReader.ReadMsg(&packet) + //c.recvMonitor.Update(_n) if err != nil { // stopServices was invoked and we are shutting down // receiving is excpected to fail since we will close the connection @@ -701,8 +701,8 @@ type ChannelStatus struct { func (c *MConnection) Status() ConnectionStatus { var status ConnectionStatus status.Duration = time.Since(c.created) - status.SendMonitor = c.sendMonitor.Status() - status.RecvMonitor = c.recvMonitor.Status() + //status.SendMonitor = c.sendMonitor.Status() + //status.RecvMonitor = c.recvMonitor.Status() status.Channels = make([]ChannelStatus, len(c.channels)) for i, channel := range c.channels { status.Channels[i] = ChannelStatus{ @@ -856,10 +856,10 @@ func (ch *Channel) writePacketMsgTo(w io.Writer) (n int, err error) { // Not goroutine-safe func (ch *Channel) recvPacketMsg(packet tmp2p.PacketMsg) ([]byte, error) { ch.Logger.Debug("Read PacketMsg", "conn", ch.conn, "packet", packet) - var recvCap, recvReceived = ch.desc.RecvMessageCapacity, len(ch.recving) + len(packet.Data) - if recvCap < recvReceived { - return nil, fmt.Errorf("received message exceeds available capacity: %v < %v", recvCap, recvReceived) - } + //var recvCap, recvReceived = ch.desc.RecvMessageCapacity, len(ch.recving) + len(packet.Data) + //if recvCap < recvReceived { + // return nil, fmt.Errorf("received message exceeds available capacity: %v < %v", recvCap, recvReceived) + //} ch.recving = append(ch.recving, packet.Data...) if packet.EOF { msgBytes := ch.recving diff --git a/p2p/conn_set.go b/p2p/conn_set.go index 0a568c37f6..1fa5547540 100644 --- a/p2p/conn_set.go +++ b/p2p/conn_set.go @@ -1,6 +1,7 @@ package p2p import ( + "github.com/quic-go/quic-go" "net" cmtsync "github.com/tendermint/tendermint/libs/sync" @@ -8,15 +9,15 @@ import ( // ConnSet is a lookup table for connections and all their ips. type ConnSet interface { - Has(net.Conn) bool + Has(quic.Connection) bool HasIP(net.IP) bool - Set(net.Conn, []net.IP) - Remove(net.Conn) + Set(quic.Connection, []net.IP) + Remove(quic.Connection) RemoveAddr(net.Addr) } type connSetItem struct { - conn net.Conn + conn quic.Connection ips []net.IP } @@ -33,7 +34,7 @@ func NewConnSet() ConnSet { } } -func (cs *connSet) Has(c net.Conn) bool { +func (cs *connSet) Has(c quic.Connection) bool { cs.RLock() defer cs.RUnlock() @@ -57,7 +58,7 @@ func (cs *connSet) HasIP(ip net.IP) bool { return false } -func (cs *connSet) Remove(c net.Conn) { +func (cs *connSet) Remove(c quic.Connection) { cs.Lock() defer cs.Unlock() @@ -71,7 +72,7 @@ func (cs *connSet) RemoveAddr(addr net.Addr) { delete(cs.conns, addr.String()) } -func (cs *connSet) Set(c net.Conn, ips []net.IP) { +func (cs *connSet) Set(c quic.Connection, ips []net.IP) { cs.Lock() defer cs.Unlock() diff --git a/p2p/errors.go b/p2p/errors.go index 4fc915292f..07377fc675 100644 --- a/p2p/errors.go +++ b/p2p/errors.go @@ -2,6 +2,7 @@ package p2p import ( "fmt" + "github.com/quic-go/quic-go" "net" ) @@ -16,7 +17,7 @@ func (e ErrFilterTimeout) Error() string { // information as to the reason. type ErrRejected struct { addr NetAddress - conn net.Conn + conn quic.Connection err error id ID isAuthFailure bool diff --git a/p2p/netaddress.go b/p2p/netaddress.go index b8e0c24196..ac73e72791 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -5,10 +5,13 @@ package p2p import ( + "context" + "crypto/tls" "encoding/hex" "errors" "flag" "fmt" + "github.com/quic-go/quic-go" "net" "strconv" "strings" @@ -232,8 +235,34 @@ func (na *NetAddress) DialString() string { } // Dial calls net.Dial on the address. -func (na *NetAddress) Dial() (net.Conn, error) { - conn, err := net.Dial("tcp", na.DialString()) +func (na *NetAddress) Dial() (quic.Connection, error) { + tlsConfig := tls.Config{ + MinVersion: tls.VersionTLS12, + } + quickConfig := quic.Config{ + GetConfigForClient: nil, + Versions: nil, + HandshakeIdleTimeout: 0, + MaxIdleTimeout: 0, + TokenStore: nil, + InitialStreamReceiveWindow: 0, + MaxStreamReceiveWindow: 0, + InitialConnectionReceiveWindow: 0, + MaxConnectionReceiveWindow: 0, + AllowConnectionWindowIncrease: nil, + MaxIncomingStreams: 0, + MaxIncomingUniStreams: 0, + KeepAlivePeriod: 0, + InitialPacketSize: 0, + DisablePathMTUDiscovery: false, + Allow0RTT: false, + EnableDatagrams: false, + Tracer: nil, + } + conn, err := quic.DialAddr(context.Background(), na.DialString(), &tlsConfig, &quickConfig) + if err != nil { + return nil, err + } if err != nil { return nil, err } @@ -241,8 +270,32 @@ func (na *NetAddress) Dial() (net.Conn, error) { } // DialTimeout calls net.DialTimeout on the address. -func (na *NetAddress) DialTimeout(timeout time.Duration) (net.Conn, error) { - conn, err := net.DialTimeout("tcp", na.DialString(), timeout) +func (na *NetAddress) DialTimeout(timeout time.Duration) (quic.Connection, error) { + tlsConfig := tls.Config{ + // TLS config should come from somewhere with a private key: mt.nodeKey.PrivKey + MinVersion: tls.VersionTLS13, + } + quickConfig := quic.Config{ + GetConfigForClient: nil, + Versions: nil, + HandshakeIdleTimeout: 0, + MaxIdleTimeout: 0, + TokenStore: nil, + InitialStreamReceiveWindow: 0, + MaxStreamReceiveWindow: 0, + InitialConnectionReceiveWindow: 0, + MaxConnectionReceiveWindow: 0, + AllowConnectionWindowIncrease: nil, + MaxIncomingStreams: 0, + MaxIncomingUniStreams: 0, + KeepAlivePeriod: 0, + InitialPacketSize: 0, + DisablePathMTUDiscovery: false, + Allow0RTT: false, + EnableDatagrams: false, + Tracer: nil, + } + conn, err := quic.DialAddr(context.Background(), na.DialString(), &tlsConfig, &quickConfig) if err != nil { return nil, err } diff --git a/p2p/peer.go b/p2p/peer.go index f43cff9d51..67e384a42c 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -2,8 +2,8 @@ package p2p import ( "fmt" + "github.com/quic-go/quic-go" "net" - "reflect" "time" "github.com/gogo/protobuf/proto" @@ -12,9 +12,6 @@ import ( "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/libs/service" "github.com/tendermint/tendermint/pkg/trace" - "github.com/tendermint/tendermint/pkg/trace/schema" - - cmtconn "github.com/tendermint/tendermint/p2p/conn" ) //go:generate ../scripts/mockery_generate.sh Peer @@ -35,7 +32,7 @@ type Peer interface { CloseConn() error // close original connection NodeInfo() NodeInfo // peer's info - Status() cmtconn.ConnectionStatus + //Status() cmtconn.ConnectionStatus SocketAddr() *NetAddress // actual address of the socket // Deprecated: entities looking to act as peers should implement SendEnvelope instead. @@ -108,7 +105,7 @@ func TrySendEnvelopeShim(p Peer, e Envelope, lg log.Logger) bool { type peerConn struct { outbound bool persistent bool - conn net.Conn // source connection + conn quic.Connection // source connection socketAddr *NetAddress @@ -118,7 +115,7 @@ type peerConn struct { func newPeerConn( outbound, persistent bool, - conn net.Conn, + conn quic.Connection, socketAddr *NetAddress, ) peerConn { @@ -133,7 +130,7 @@ func newPeerConn( // ID only exists for SecretConnection. // NOTE: Will panic if conn is not *SecretConnection. func (pc peerConn) ID() ID { - return PubKeyToID(pc.conn.(*cmtconn.SecretConnection).RemotePubKey()) + return ID(pc.conn.RemoteAddr().String()) } // Return the IP from the connection RemoteAddr @@ -165,7 +162,6 @@ type peer struct { // raw peerConn and the multiplex connection peerConn - mconn *cmtconn.MConnection // peer's node info and the channel it knows about // channels = nodeInfo.Channels @@ -173,6 +169,8 @@ type peer struct { nodeInfo NodeInfo channels []byte + streams map[byte]quic.Stream + // User data Data *cmap.CMap @@ -195,15 +193,11 @@ func WithPeerTracer(t trace.Tracer) PeerOption { func newPeer( pc peerConn, - mConfig cmtconn.MConnConfig, nodeInfo NodeInfo, - reactorsByCh map[byte]Reactor, - msgTypeByChID map[byte]proto.Message, - chDescs []*cmtconn.ChannelDescriptor, - onPeerError func(Peer, interface{}), mlc *metricsLabelCache, options ...PeerOption, ) *peer { + pc.conn.OpenStream() p := &peer{ peerConn: pc, nodeInfo: nodeInfo, @@ -213,17 +207,9 @@ func newPeer( metrics: NopMetrics(), mlc: mlc, traceClient: trace.NoOpTracer(), + streams: make(map[byte]quic.Stream), } - p.mconn = createMConnection( - pc.conn, - p, - reactorsByCh, - msgTypeByChID, - chDescs, - onPeerError, - mConfig, - ) p.BaseService = *service.NewBaseService(nil, "Peer", p) for _, option := range options { option(p) @@ -235,10 +221,10 @@ func newPeer( // String representation. func (p *peer) String() string { if p.outbound { - return fmt.Sprintf("Peer{%v %v out}", p.mconn, p.ID()) + return fmt.Sprintf("Peer{%v %v out}", p.conn, p.ID()) } - return fmt.Sprintf("Peer{%v %v in}", p.mconn, p.ID()) + return fmt.Sprintf("Peer{%v %v in}", p.conn, p.ID()) } //--------------------------------------------------- @@ -247,7 +233,6 @@ func (p *peer) String() string { // SetLogger implements BaseService. func (p *peer) SetLogger(l log.Logger) { p.Logger = l - p.mconn.SetLogger(l) } // OnStart implements BaseService. @@ -256,10 +241,6 @@ func (p *peer) OnStart() error { return err } - if err := p.mconn.Start(); err != nil { - return err - } - go p.metricsReporter() return nil } @@ -270,14 +251,17 @@ func (p *peer) OnStart() error { func (p *peer) FlushStop() { p.metricsTicker.Stop() p.BaseService.OnStop() - p.mconn.FlushStop() // stop everything and close the conn + for _, stream := range p.streams { + stream.CancelRead(quic.StreamErrorCode(1)) // stop everything and close the conn + stream.CancelWrite(quic.StreamErrorCode(1)) // stop everything and close the conn + } } // OnStop implements BaseService. func (p *peer) OnStop() { p.metricsTicker.Stop() p.BaseService.OnStop() - if err := p.mconn.Stop(); err != nil { // stop everything and close the conn + if err := p.conn.CloseWithError(quic.ApplicationErrorCode(2), "stopping in peer.OnStop()"); err != nil { // stop everything and close the conn p.Logger.Debug("Error while stopping peer", "err", err) } } @@ -292,12 +276,12 @@ func (p *peer) ID() ID { // IsOutbound returns true if the connection is outbound, false otherwise. func (p *peer) IsOutbound() bool { - return p.peerConn.outbound + return true } // IsPersistent returns true if the peer is persitent, false otherwise. func (p *peer) IsPersistent() bool { - return p.peerConn.persistent + return true } // NodeInfo returns a copy of the peer's NodeInfo. @@ -310,13 +294,13 @@ func (p *peer) NodeInfo() NodeInfo { // For inbound peers, it's the address returned by the underlying connection // (not what's reported in the peer's NodeInfo). func (p *peer) SocketAddr() *NetAddress { - return p.peerConn.socketAddr + return p.socketAddr } -// Status returns the peer's ConnectionStatus. -func (p *peer) Status() cmtconn.ConnectionStatus { - return p.mconn.Status() -} +//// Status returns the peer's ConnectionStatus. +//func (p *peer) Status() cmtconn.ConnectionStatus { +// return cmtconn.ConnectionStatus{} +//} // SendEnvelope sends the message in the envelope on the channel specified by the // envelope. Returns false if the connection times out trying to place the message @@ -360,49 +344,27 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { } else if !p.hasChannel(chID) { return false } - res := p.mconn.Send(chID, msgBytes) - if res { - labels := []string{ - "peer_id", string(p.ID()), - "chID", fmt.Sprintf("%#x", chID), + stream, has := p.streams[chID] + if !has { + newStream, err := p.conn.OpenStream() + if err != nil { + p.Logger.Error("error opening quic stream", "err", err.Error()) + return false } - p.metrics.PeerSendBytesTotal.With(labels...).Add(float64(len(msgBytes))) + p.streams[chID] = newStream + stream = newStream } - return res -} -// TrySendEnvelope attempts to sends the message in the envelope on the channel specified by the -// envelope. Returns false immediately if the connection's internal queue is full -// Using TrySendEnvelope allows for tracking the message bytes sent and received by message type -// as a metric which TrySend cannot support. -func (p *peer) TrySendEnvelope(e Envelope) bool { - if !p.IsRunning() { - // see Switch#Broadcast, where we fetch the list of peers and loop over - // them - while we're looping, one peer may be removed and stopped. - return false - } else if !p.hasChannel(e.ChannelID) { - return false - } - msg := e.Message - metricLabelValue := p.mlc.ValueToMetricLabel(msg) - if w, ok := msg.(Wrapper); ok { - msg = w.Wrap() - } - msgBytes, err := proto.Marshal(msg) + n, err := stream.Write(msgBytes) if err != nil { - p.Logger.Error("marshaling message to send", "error", err) return false } - res := p.TrySend(e.ChannelID, msgBytes) - if res { - labels := []string{ - "message_type", metricLabelValue, - "chID", fmt.Sprintf("%#x", e.ChannelID), - "peer_id", string(p.ID()), - } - p.metrics.MessageSendBytesTotal.With(labels...).Add(float64(len(msgBytes))) + labels := []string{ + "peer_id", string(p.ID()), + "chID", fmt.Sprintf("%#x", chID), } - return res + p.metrics.PeerSendBytesTotal.With(labels...).Add(float64(n)) + return true } // TrySend msg bytes to the channel identified by chID byte. Immediately returns @@ -414,7 +376,7 @@ func (p *peer) TrySend(chID byte, msgBytes []byte) bool { } else if !p.hasChannel(chID) { return false } - res := p.mconn.TrySend(chID, msgBytes) + res := p.Send(chID, msgBytes) if res { labels := []string{ "peer_id", string(p.ID()), @@ -457,7 +419,7 @@ func (p *peer) hasChannel(chID byte) bool { // CloseConn closes original connection. Used for cleaning up in cases where the peer had not been started at all. func (p *peer) CloseConn() error { - return p.peerConn.conn.Close() + return p.conn.CloseWithError(quic.ApplicationErrorCode(1), "closed in peer.CloseConn()") } func (p *peer) SetRemovalFailed() { @@ -474,12 +436,12 @@ func (p *peer) GetRemovalFailed() bool { // CloseConn closes the underlying connection func (pc *peerConn) CloseConn() { - pc.conn.Close() + pc.conn.CloseWithError(quic.ApplicationErrorCode(1), "closed in CloseConn") } // RemoteAddr returns peer's remote network address. func (p *peer) RemoteAddr() net.Addr { - return p.peerConn.conn.RemoteAddr() + return p.conn.RemoteAddr() } // CanSend returns true if the send queue is not full, false otherwise. @@ -487,7 +449,7 @@ func (p *peer) CanSend(chID byte) bool { if !p.IsRunning() { return false } - return p.mconn.CanSend(chID) + return true } //--------------------------------------------------- @@ -502,84 +464,9 @@ func (p *peer) metricsReporter() { for { select { case <-p.metricsTicker.C: - status := p.mconn.Status() - var sendQueueSize float64 - queues := make(map[byte]int, len(status.Channels)) - for _, chStatus := range status.Channels { - sendQueueSize += float64(chStatus.SendQueueSize) - queues[chStatus.ID] = chStatus.SendQueueSize - } - - p.metrics.PeerPendingSendBytes.With("peer_id", string(p.ID())).Set(sendQueueSize) - schema.WritePendingBytes(p.traceClient, string(p.ID()), queues) + fmt.Println("in metrics reporter <-p.metricsTicker.C") case <-p.Quit(): return } } } - -//------------------------------------------------------------------ -// helper funcs - -func createMConnection( - conn net.Conn, - p *peer, - reactorsByCh map[byte]Reactor, - msgTypeByChID map[byte]proto.Message, - chDescs []*cmtconn.ChannelDescriptor, - onPeerError func(Peer, interface{}), - config cmtconn.MConnConfig, -) *cmtconn.MConnection { - - onReceive := func(chID byte, msgBytes []byte) { - reactor := reactorsByCh[chID] - if reactor == nil { - // Note that its ok to panic here as it's caught in the conn._recover, - // which does onPeerError. - panic(fmt.Sprintf("Unknown channel %X", chID)) - } - mt := msgTypeByChID[chID] - msg := proto.Clone(mt) - err := proto.Unmarshal(msgBytes, msg) - if err != nil { - panic(fmt.Errorf("unmarshaling message: %s into type: %s", err, reflect.TypeOf(mt))) - } - - if w, ok := msg.(Unwrapper); ok { - msg, err = w.Unwrap() - if err != nil { - panic(fmt.Errorf("unwrapping message: %s", err)) - } - } - - labels := []string{ - "peer_id", string(p.ID()), - "chID", fmt.Sprintf("%#x", chID), - } - - p.metrics.PeerReceiveBytesTotal.With(labels...).Add(float64(len(msgBytes))) - p.metrics.MessageReceiveBytesTotal.With(append(labels, "message_type", p.mlc.ValueToMetricLabel(msg))...).Add(float64(len(msgBytes))) - schema.WriteReceivedBytes(p.traceClient, string(p.ID()), chID, len(msgBytes)) - if nr, ok := reactor.(EnvelopeReceiver); ok { - nr.ReceiveEnvelope(Envelope{ - ChannelID: chID, - Src: p, - Message: msg, - }) - } else { - reactor.Receive(chID, p, msgBytes) - } - } - - onError := func(r interface{}) { - onPeerError(p, r) - } - - return cmtconn.NewMConnectionWithConfig( - conn, - chDescs, - onReceive, - onError, - config, - ) -} diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 5fc24bfca2..8883728a6b 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -763,9 +763,9 @@ func (r *Reactor) cleanupCrawlPeerInfos() { // attemptDisconnects checks if we've been with each peer long enough to disconnect func (r *Reactor) attemptDisconnects() { for _, peer := range r.Switch.Peers().List() { - if peer.Status().Duration < r.config.SeedDisconnectWaitPeriod { - continue - } + //if peer.Status().Duration < r.config.SeedDisconnectWaitPeriod { + // continue + //} if peer.IsPersistent() { continue } diff --git a/p2p/test_util.go b/p2p/test_util.go index c527863054..8c41148631 100644 --- a/p2p/test_util.go +++ b/p2p/test_util.go @@ -1,3 +1,5 @@ +//go:build exclude + package p2p import ( diff --git a/p2p/transport.go b/p2p/transport.go index 5c82f1f757..c4b8546ae7 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -2,18 +2,15 @@ package p2p import ( "context" + "crypto/tls" "fmt" + "github.com/quic-go/quic-go" "net" "time" - "golang.org/x/net/netutil" - "github.com/gogo/protobuf/proto" - "github.com/tendermint/tendermint/crypto" - "github.com/tendermint/tendermint/libs/protoio" "github.com/tendermint/tendermint/p2p/conn" "github.com/tendermint/tendermint/pkg/trace" - tmp2p "github.com/tendermint/tendermint/proto/tendermint/p2p" ) const ( @@ -31,7 +28,7 @@ type IPResolver interface { // asynchronously running routine to the Accept method. type accept struct { netAddr *NetAddress - conn net.Conn + conn quic.Connection nodeInfo NodeInfo err error } @@ -83,12 +80,12 @@ type transportLifecycle interface { // ConnFilterFunc to be implemented by filter hooks after a new connection has // been established. The set of exisiting connections is passed along together // with all resolved IPs for the new connection. -type ConnFilterFunc func(ConnSet, net.Conn, []net.IP) error +type ConnFilterFunc func(ConnSet, quic.Connection, []net.IP) error // ConnDuplicateIPFilter resolves and keeps all ips for an incoming connection // and refuses new ones if they come from a known ip. func ConnDuplicateIPFilter() ConnFilterFunc { - return func(cs ConnSet, c net.Conn, ips []net.IP) error { + return func(cs ConnSet, c quic.Connection, ips []net.IP) error { for _, ip := range ips { if cs.HasIP(ip) { return ErrRejected{ @@ -138,7 +135,7 @@ func MultiplexTransportMaxIncomingConnections(n int) MultiplexTransportOption { // multiplexed peers. type MultiplexTransport struct { netAddr NetAddress - listener net.Listener + listener *quic.Listener maxIncomingConnections int // see MaxIncomingConnections acceptc chan accept @@ -253,19 +250,38 @@ func (mt *MultiplexTransport) Close() error { // Listen implements transportLifecycle. func (mt *MultiplexTransport) Listen(addr NetAddress) error { - ln, err := net.Listen("tcp", addr.DialString()) + tlsConfig := tls.Config{ + MinVersion: tls.VersionTLS13, + } + quickConfig := quic.Config{ + GetConfigForClient: nil, + Versions: nil, + HandshakeIdleTimeout: 0, + MaxIdleTimeout: 0, + TokenStore: nil, + InitialStreamReceiveWindow: 0, + MaxStreamReceiveWindow: 0, + InitialConnectionReceiveWindow: 0, + MaxConnectionReceiveWindow: 0, + AllowConnectionWindowIncrease: nil, + MaxIncomingStreams: 0, + MaxIncomingUniStreams: 0, + KeepAlivePeriod: 0, + InitialPacketSize: 0, + DisablePathMTUDiscovery: false, + Allow0RTT: false, + EnableDatagrams: false, + Tracer: nil, + } + listener, err := quic.ListenAddr(addr.DialString(), &tlsConfig, &quickConfig) if err != nil { return err } - - if mt.maxIncomingConnections > 0 { - ln = netutil.LimitListener(ln, mt.maxIncomingConnections) - } - + mt.listener = listener mt.netAddr = addr - mt.listener = ln - go mt.acceptPeers() + // TODO use a better context + go mt.acceptPeers(context.Background()) return nil } @@ -283,9 +299,9 @@ func (mt *MultiplexTransport) AddChannel(chID byte) { } } -func (mt *MultiplexTransport) acceptPeers() { +func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { for { - c, err := mt.listener.Accept() + c, err := mt.listener.Accept(ctx) if err != nil { // If Close() has been called, silently exit. select { @@ -306,7 +322,7 @@ func (mt *MultiplexTransport) acceptPeers() { // Reference: https://github.com/tendermint/tendermint/issues/2047 // // [0] https://en.wikipedia.org/wiki/Head-of-line_blocking - go func(c net.Conn) { + go func(c quic.Connection) { defer func() { if r := recover(); r != nil { err := ErrRejected{ @@ -318,34 +334,28 @@ func (mt *MultiplexTransport) acceptPeers() { case mt.acceptc <- accept{err: err}: case <-mt.closec: // Give up if the transport was closed. - _ = c.Close() + _ = c.CloseWithError(quic.ApplicationErrorCode(1), "some error") return } } }() var ( - nodeInfo NodeInfo - secretConn *conn.SecretConnection - netAddr *NetAddress + nodeInfo NodeInfo + netAddr *NetAddress ) err := mt.filterConn(c) - if err == nil { - secretConn, nodeInfo, err = mt.upgrade(c, nil) - if err == nil { - addr := c.RemoteAddr() - id := PubKeyToID(secretConn.RemotePubKey()) - netAddr = NewNetAddress(id, addr) - } + if err != nil { + return } select { - case mt.acceptc <- accept{netAddr, secretConn, nodeInfo, err}: + case mt.acceptc <- accept{netAddr, c, nodeInfo, err}: // Make the upgraded peer available. case <-mt.closec: // Give up if the transport was closed. - _ = c.Close() + _ = c.CloseWithError(quic.ApplicationErrorCode(1), "some error") return } }(c) @@ -359,16 +369,16 @@ func (mt *MultiplexTransport) Cleanup(p Peer) { _ = p.CloseConn() } -func (mt *MultiplexTransport) cleanup(c net.Conn) error { +func (mt *MultiplexTransport) cleanup(c quic.Connection) error { mt.conns.Remove(c) - return c.Close() + return c.CloseWithError(quic.ApplicationErrorCode(1), "some error") } -func (mt *MultiplexTransport) filterConn(c net.Conn) (err error) { +func (mt *MultiplexTransport) filterConn(c quic.Connection) (err error) { defer func() { if err != nil { - _ = c.Close() + _ = c.CloseWithError(quic.ApplicationErrorCode(1), "some error") } }() @@ -386,7 +396,7 @@ func (mt *MultiplexTransport) filterConn(c net.Conn) (err error) { errc := make(chan error, len(mt.connFilters)) for _, f := range mt.connFilters { - go func(f ConnFilterFunc, c net.Conn, ips []net.IP, errc chan<- error) { + go func(f ConnFilterFunc, c quic.Connection, ips []net.IP, errc chan<- error) { errc <- f(mt.conns, c, ips) }(f, c, ips, errc) } @@ -409,96 +419,16 @@ func (mt *MultiplexTransport) filterConn(c net.Conn) (err error) { } func (mt *MultiplexTransport) upgrade( - c net.Conn, + c quic.Connection, dialedAddr *NetAddress, -) (secretConn *conn.SecretConnection, nodeInfo NodeInfo, err error) { - defer func() { - if err != nil { - _ = mt.cleanup(c) - } - }() - - secretConn, err = upgradeSecretConn(c, mt.handshakeTimeout, mt.nodeKey.PrivKey) - if err != nil { - return nil, nil, ErrRejected{ - conn: c, - err: fmt.Errorf("secret conn failed: %v", err), - isAuthFailure: true, - } - } - - // For outgoing conns, ensure connection key matches dialed key. - connID := PubKeyToID(secretConn.RemotePubKey()) - if dialedAddr != nil { - if dialedID := dialedAddr.ID; connID != dialedID { - return nil, nil, ErrRejected{ - conn: c, - id: connID, - err: fmt.Errorf( - "conn.ID (%v) dialed ID (%v) mismatch", - connID, - dialedID, - ), - isAuthFailure: true, - } - } - } - - nodeInfo, err = handshake(secretConn, mt.handshakeTimeout, mt.nodeInfo) - if err != nil { - return nil, nil, ErrRejected{ - conn: c, - err: fmt.Errorf("handshake failed: %v", err), - isAuthFailure: true, - } - } - - if err := nodeInfo.Validate(); err != nil { - return nil, nil, ErrRejected{ - conn: c, - err: err, - isNodeInfoInvalid: true, - } - } - - // Ensure connection key matches self reported key. - if connID != nodeInfo.ID() { - return nil, nil, ErrRejected{ - conn: c, - id: connID, - err: fmt.Errorf( - "conn.ID (%v) NodeInfo.ID (%v) mismatch", - connID, - nodeInfo.ID(), - ), - isAuthFailure: true, - } - } - - // Reject self. - if mt.nodeInfo.ID() == nodeInfo.ID() { - return nil, nil, ErrRejected{ - addr: *NewNetAddress(nodeInfo.ID(), c.RemoteAddr()), - conn: c, - id: nodeInfo.ID(), - isSelf: true, - } - } - - if err := mt.nodeInfo.CompatibleWith(nodeInfo); err != nil { - return nil, nil, ErrRejected{ - conn: c, - err: err, - id: nodeInfo.ID(), - isIncompatible: true, - } - } - - return secretConn, nodeInfo, nil +) (conn quic.Connection, nodeInfo NodeInfo, err error) { + // TODO should be removed since we don't need to upgrade a + // Quic connection + return c, nodeInfo, nil } func (mt *MultiplexTransport) wrapPeer( - c net.Conn, + c quic.Connection, ni NodeInfo, cfg peerConfig, socketAddr *NetAddress, @@ -525,12 +455,7 @@ func (mt *MultiplexTransport) wrapPeer( p := newPeer( peerConn, - mt.mConfig, ni, - cfg.reactorsByCh, - cfg.msgTypeByChID, - cfg.chDescs, - cfg.onPeerError, cfg.mlc, PeerMetrics(cfg.metrics), WithPeerTracer(mt.tracer), @@ -539,66 +464,7 @@ func (mt *MultiplexTransport) wrapPeer( return p } -func handshake( - c net.Conn, - timeout time.Duration, - nodeInfo NodeInfo, -) (NodeInfo, error) { - if err := c.SetDeadline(time.Now().Add(timeout)); err != nil { - return nil, err - } - - var ( - errc = make(chan error, 2) - - pbpeerNodeInfo tmp2p.DefaultNodeInfo - peerNodeInfo DefaultNodeInfo - ourNodeInfo = nodeInfo.(DefaultNodeInfo) - ) - - go func(errc chan<- error, c net.Conn) { - _, err := protoio.NewDelimitedWriter(c).WriteMsg(ourNodeInfo.ToProto()) - errc <- err - }(errc, c) - go func(errc chan<- error, c net.Conn) { - protoReader := protoio.NewDelimitedReader(c, MaxNodeInfoSize()) - _, err := protoReader.ReadMsg(&pbpeerNodeInfo) - errc <- err - }(errc, c) - - for i := 0; i < cap(errc); i++ { - err := <-errc - if err != nil { - return nil, err - } - } - - peerNodeInfo, err := DefaultNodeInfoFromToProto(&pbpeerNodeInfo) - if err != nil { - return nil, err - } - - return peerNodeInfo, c.SetDeadline(time.Time{}) -} - -func upgradeSecretConn( - c net.Conn, - timeout time.Duration, - privKey crypto.PrivKey, -) (*conn.SecretConnection, error) { - if err := c.SetDeadline(time.Now().Add(timeout)); err != nil { - return nil, err - } - - sc, err := conn.MakeSecretConnection(c, privKey) - if err != nil { - return nil, err - } - - return sc, sc.SetDeadline(time.Time{}) -} - -func resolveIPs(resolver IPResolver, c net.Conn) ([]net.IP, error) { +func resolveIPs(resolver IPResolver, c quic.Connection) ([]net.IP, error) { host, _, err := net.SplitHostPort(c.RemoteAddr().String()) if err != nil { return nil, err diff --git a/rpc/core/net.go b/rpc/core/net.go index 57ae4f1b42..1d07d594aa 100644 --- a/rpc/core/net.go +++ b/rpc/core/net.go @@ -22,10 +22,10 @@ func NetInfo(ctx *rpctypes.Context) (*ctypes.ResultNetInfo, error) { return nil, fmt.Errorf("peer.NodeInfo() is not DefaultNodeInfo") } peers = append(peers, ctypes.Peer{ - NodeInfo: nodeInfo, - IsOutbound: peer.IsOutbound(), - ConnectionStatus: peer.Status(), - RemoteIP: peer.RemoteIP().String(), + NodeInfo: nodeInfo, + IsOutbound: peer.IsOutbound(), + //ConnectionStatus: peer.Status(), + RemoteIP: peer.RemoteIP().String(), }) } // TODO: Should we include PersistentPeers and Seeds in here? From 89de565feb47207a755376c6ca28bdca01dfed1e Mon Sep 17 00:00:00 2001 From: sweexordious Date: Tue, 13 Aug 2024 11:50:39 +0100 Subject: [PATCH 02/55] working connection but not receiving data --- libs/net/net.go | 23 +----- node/node.go | 1 + p2p/netaddress.go | 79 +++++-------------- p2p/peer.go | 80 ++++++++++++++++++- p2p/transport.go | 190 +++++++++++++++++++++++++++++++++++++++------- 5 files changed, 259 insertions(+), 114 deletions(-) diff --git a/libs/net/net.go b/libs/net/net.go index f27e8a3214..a911dca129 100644 --- a/libs/net/net.go +++ b/libs/net/net.go @@ -22,29 +22,10 @@ func Connect(protoAddr string) (net.Conn, error) { func QuicConnect(protoAddr string) (quic.Connection, error) { proto, address := ProtocolAndAddress(protoAddr) tlsConfig := tls.Config{ - MinVersion: tls.VersionTLS12, + InsecureSkipVerify: true, } // TODO better config - quickConfig := quic.Config{ - GetConfigForClient: nil, - Versions: nil, - HandshakeIdleTimeout: 0, - MaxIdleTimeout: 0, - TokenStore: nil, - InitialStreamReceiveWindow: 0, - MaxStreamReceiveWindow: 0, - InitialConnectionReceiveWindow: 0, - MaxConnectionReceiveWindow: 0, - AllowConnectionWindowIncrease: nil, - MaxIncomingStreams: 0, - MaxIncomingUniStreams: 0, - KeepAlivePeriod: 0, - InitialPacketSize: 0, - DisablePathMTUDiscovery: false, - Allow0RTT: false, - EnableDatagrams: false, - Tracer: nil, - } + quickConfig := quic.Config{} conn, err := quic.DialAddr(context.Background(), fmt.Sprintf("%s:%s", proto, address), &tlsConfig, &quickConfig) return conn, err } diff --git a/node/node.go b/node/node.go index 24b6cdd3a0..1827609db7 100644 --- a/node/node.go +++ b/node/node.go @@ -1059,6 +1059,7 @@ func (n *Node) OnStart() error { if err != nil { return err } + addr.PrivateKey = n.nodeKey.PrivKey if err := n.transport.Listen(*addr); err != nil { return err } diff --git a/p2p/netaddress.go b/p2p/netaddress.go index ac73e72791..4ade8bf65e 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -9,9 +9,9 @@ import ( "crypto/tls" "encoding/hex" "errors" - "flag" "fmt" "github.com/quic-go/quic-go" + "github.com/tendermint/tendermint/crypto" "net" "strconv" "strings" @@ -26,9 +26,10 @@ const EmptyNetAddress = "" // NetAddress defines information about a peer on the network // including its ID, IP address, and port. type NetAddress struct { - ID ID `json:"id"` - IP net.IP `json:"ip"` - Port uint16 `json:"port"` + ID ID `json:"id"` + IP net.IP `json:"ip"` + Port uint16 `json:"port"` + PrivateKey crypto.PrivKey } // IDAddressString returns id@hostPort. It strips the leading @@ -44,16 +45,16 @@ func IDAddressString(id ID, protocolHostPort string) string { // panic. Panics if ID is invalid. // TODO: socks proxies? func NewNetAddress(id ID, addr net.Addr) *NetAddress { - tcpAddr, ok := addr.(*net.TCPAddr) - if !ok { - if flag.Lookup("test.v") == nil { // normal run - panic(fmt.Sprintf("Only TCPAddrs are supported. Got: %v", addr)) - } else { // in testing - netAddr := NewNetAddressIPPort(net.IP("127.0.0.1"), 0) - netAddr.ID = id - return netAddr - } - } + tcpAddr := addr.(*net.UDPAddr) + //if !ok { + // if flag.Lookup("test.v") == nil { // normal run + // panic(fmt.Sprintf("Only TCPAddrs are supported. Got: %v", addr)) + // } else { // in testing + // netAddr := NewNetAddressIPPort(net.IP("127.0.0.1"), 0) + // netAddr.ID = id + // return netAddr + // } + //} if err := validateID(id); err != nil { panic(fmt.Sprintf("Invalid ID %v: %v (addr: %v)", id, err, addr)) @@ -237,64 +238,22 @@ func (na *NetAddress) DialString() string { // Dial calls net.Dial on the address. func (na *NetAddress) Dial() (quic.Connection, error) { tlsConfig := tls.Config{ - MinVersion: tls.VersionTLS12, - } - quickConfig := quic.Config{ - GetConfigForClient: nil, - Versions: nil, - HandshakeIdleTimeout: 0, - MaxIdleTimeout: 0, - TokenStore: nil, - InitialStreamReceiveWindow: 0, - MaxStreamReceiveWindow: 0, - InitialConnectionReceiveWindow: 0, - MaxConnectionReceiveWindow: 0, - AllowConnectionWindowIncrease: nil, - MaxIncomingStreams: 0, - MaxIncomingUniStreams: 0, - KeepAlivePeriod: 0, - InitialPacketSize: 0, - DisablePathMTUDiscovery: false, - Allow0RTT: false, - EnableDatagrams: false, - Tracer: nil, + InsecureSkipVerify: true, } + quickConfig := quic.Config{} conn, err := quic.DialAddr(context.Background(), na.DialString(), &tlsConfig, &quickConfig) if err != nil { return nil, err } - if err != nil { - return nil, err - } return conn, nil } // DialTimeout calls net.DialTimeout on the address. func (na *NetAddress) DialTimeout(timeout time.Duration) (quic.Connection, error) { tlsConfig := tls.Config{ - // TLS config should come from somewhere with a private key: mt.nodeKey.PrivKey - MinVersion: tls.VersionTLS13, - } - quickConfig := quic.Config{ - GetConfigForClient: nil, - Versions: nil, - HandshakeIdleTimeout: 0, - MaxIdleTimeout: 0, - TokenStore: nil, - InitialStreamReceiveWindow: 0, - MaxStreamReceiveWindow: 0, - InitialConnectionReceiveWindow: 0, - MaxConnectionReceiveWindow: 0, - AllowConnectionWindowIncrease: nil, - MaxIncomingStreams: 0, - MaxIncomingUniStreams: 0, - KeepAlivePeriod: 0, - InitialPacketSize: 0, - DisablePathMTUDiscovery: false, - Allow0RTT: false, - EnableDatagrams: false, - Tracer: nil, + InsecureSkipVerify: true, } + quickConfig := quic.Config{} conn, err := quic.DialAddr(context.Background(), na.DialString(), &tlsConfig, &quickConfig) if err != nil { return nil, err diff --git a/p2p/peer.go b/p2p/peer.go index 67e384a42c..197f6e683a 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -197,7 +197,6 @@ func newPeer( mlc *metricsLabelCache, options ...PeerOption, ) *peer { - pc.conn.OpenStream() p := &peer{ peerConn: pc, nodeInfo: nodeInfo, @@ -210,6 +209,16 @@ func newPeer( streams: make(map[byte]quic.Stream), } + //p.mconn = createMConnection( + // pc.conn, + // p, + // reactorsByCh, + // msgTypeByChID, + // chDescs, + // onPeerError, + // mConfig, + //) + p.BaseService = *service.NewBaseService(nil, "Peer", p) for _, option := range options { option(p) @@ -276,12 +285,12 @@ func (p *peer) ID() ID { // IsOutbound returns true if the connection is outbound, false otherwise. func (p *peer) IsOutbound() bool { - return true + return p.peerConn.outbound } // IsPersistent returns true if the peer is persitent, false otherwise. func (p *peer) IsPersistent() bool { - return true + return p.peerConn.persistent } // NodeInfo returns a copy of the peer's NodeInfo. @@ -464,9 +473,72 @@ func (p *peer) metricsReporter() { for { select { case <-p.metricsTicker.C: - fmt.Println("in metrics reporter <-p.metricsTicker.C") + //fmt.Println("in metrics reporter <-p.metricsTicker.C") case <-p.Quit(): return } } } + +//func createMConnection( +// conn quic.Connection, +// p *peer, +// reactorsByCh map[byte]Reactor, +// msgTypeByChID map[byte]proto.Message, +// chDescs []*cmtconn.ChannelDescriptor, +// onPeerError func(Peer, interface{}), +// config cmtconn.MConnConfig, +//) *cmtconn.MConnection { +// +// onReceive := func(chID byte, msgBytes []byte) { +// reactor := reactorsByCh[chID] +// if reactor == nil { +// // Note that its ok to panic here as it's caught in the conn._recover, +// // which does onPeerError. +// panic(fmt.Sprintf("Unknown channel %X", chID)) +// } +// mt := msgTypeByChID[chID] +// msg := proto.Clone(mt) +// err := proto.Unmarshal(msgBytes, msg) +// if err != nil { +// panic(fmt.Errorf("unmarshaling message: %s into type: %s", err, reflect.TypeOf(mt))) +// } +// +// if w, ok := msg.(Unwrapper); ok { +// msg, err = w.Unwrap() +// if err != nil { +// panic(fmt.Errorf("unwrapping message: %s", err)) +// } +// } +// +// labels := []string{ +// "peer_id", string(p.ID()), +// "chID", fmt.Sprintf("%#x", chID), +// } +// +// p.metrics.PeerReceiveBytesTotal.With(labels...).Add(float64(len(msgBytes))) +// p.metrics.MessageReceiveBytesTotal.With(append(labels, "message_type", p.mlc.ValueToMetricLabel(msg))...).Add(float64(len(msgBytes))) +// schema.WriteReceivedBytes(p.traceClient, string(p.ID()), chID, len(msgBytes)) +// if nr, ok := reactor.(EnvelopeReceiver); ok { +// nr.ReceiveEnvelope(Envelope{ +// ChannelID: chID, +// Src: p, +// Message: msg, +// }) +// } else { +// reactor.Receive(chID, p, msgBytes) +// } +// } +// +// onError := func(r interface{}) { +// onPeerError(p, r) +// } +// +// return cmtconn.NewMConnectionWithConfig( +// conn, +// chDescs, +// onReceive, +// onError, +// config, +// ) +//} diff --git a/p2p/transport.go b/p2p/transport.go index c4b8546ae7..6b7cec76e1 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -2,12 +2,19 @@ package p2p import ( "context" + "crypto/ed25519" "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" "fmt" "github.com/quic-go/quic-go" + "github.com/tendermint/tendermint/libs/protoio" + tmp2p "github.com/tendermint/tendermint/proto/tendermint/p2p" + "math/big" "net" "time" + "crypto/rand" "github.com/gogo/protobuf/proto" "github.com/tendermint/tendermint/p2p/conn" "github.com/tendermint/tendermint/pkg/trace" @@ -215,6 +222,7 @@ func (mt *MultiplexTransport) Dial( addr NetAddress, cfg peerConfig, ) (Peer, error) { + addr.PrivateKey = mt.nodeKey.PrivKey c, err := addr.DialTimeout(mt.dialTimeout) if err != nil { return nil, err @@ -250,29 +258,29 @@ func (mt *MultiplexTransport) Close() error { // Listen implements transportLifecycle. func (mt *MultiplexTransport) Listen(addr NetAddress) error { - tlsConfig := tls.Config{ - MinVersion: tls.VersionTLS13, + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + certTemplate := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"Test"}, + }, + BasicConstraintsValid: true, } - quickConfig := quic.Config{ - GetConfigForClient: nil, - Versions: nil, - HandshakeIdleTimeout: 0, - MaxIdleTimeout: 0, - TokenStore: nil, - InitialStreamReceiveWindow: 0, - MaxStreamReceiveWindow: 0, - InitialConnectionReceiveWindow: 0, - MaxConnectionReceiveWindow: 0, - AllowConnectionWindowIncrease: nil, - MaxIncomingStreams: 0, - MaxIncomingUniStreams: 0, - KeepAlivePeriod: 0, - InitialPacketSize: 0, - DisablePathMTUDiscovery: false, - Allow0RTT: false, - EnableDatagrams: false, - Tracer: nil, + + edKey := ed25519.PrivateKey(addr.PrivateKey.Bytes()) + derBytes, err := x509.CreateCertificate(rand.Reader, &certTemplate, &certTemplate, edKey.Public(), edKey) + if err != nil { + return err + } + + tlsConfig := tls.Config{ + Certificates: []tls.Certificate{{ + Certificate: [][]byte{derBytes}, + PrivateKey: edKey, + }}, } + quickConfig := quic.Config{} listener, err := quic.ListenAddr(addr.DialString(), &tlsConfig, &quickConfig) if err != nil { return err @@ -334,7 +342,7 @@ func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { case mt.acceptc <- accept{err: err}: case <-mt.closec: // Give up if the transport was closed. - _ = c.CloseWithError(quic.ApplicationErrorCode(1), "some error") + _ = c.CloseWithError(quic.ApplicationErrorCode(1), "some error 1") return } } @@ -346,8 +354,13 @@ func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { ) err := mt.filterConn(c) - if err != nil { - return + if err == nil { + _, nodeInfo, err = mt.upgrade(c, nil) + if err == nil { + addr := c.RemoteAddr() + id := PubKeyToID(mt.nodeKey.PubKey()) + netAddr = NewNetAddress(id, addr) + } } select { @@ -355,7 +368,7 @@ func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { // Make the upgraded peer available. case <-mt.closec: // Give up if the transport was closed. - _ = c.CloseWithError(quic.ApplicationErrorCode(1), "some error") + _ = c.CloseWithError(quic.ApplicationErrorCode(1), "some error 2") return } }(c) @@ -372,13 +385,13 @@ func (mt *MultiplexTransport) Cleanup(p Peer) { func (mt *MultiplexTransport) cleanup(c quic.Connection) error { mt.conns.Remove(c) - return c.CloseWithError(quic.ApplicationErrorCode(1), "some error") + return c.CloseWithError(quic.ApplicationErrorCode(1), "some error 3") } func (mt *MultiplexTransport) filterConn(c quic.Connection) (err error) { defer func() { if err != nil { - _ = c.CloseWithError(quic.ApplicationErrorCode(1), "some error") + _ = c.CloseWithError(quic.ApplicationErrorCode(1), "some error 4") } }() @@ -422,11 +435,130 @@ func (mt *MultiplexTransport) upgrade( c quic.Connection, dialedAddr *NetAddress, ) (conn quic.Connection, nodeInfo NodeInfo, err error) { - // TODO should be removed since we don't need to upgrade a - // Quic connection + defer func() { + if err != nil { + _ = mt.cleanup(c) + } + }() + + // For outgoing conns, ensure connection key matches dialed key. + //connID := PubKeyToID(mt.nodeKey.PubKey()) + //if dialedAddr != nil { + // if dialedID := dialedAddr.ID; connID != dialedID { + // return nil, nil, ErrRejected{ + // conn: c, + // id: connID, + // err: fmt.Errorf( + // "conn.ID (%v) dialed ID (%v) mismatch", + // connID, + // dialedID, + // ), + // isAuthFailure: true, + // } + // } + //} + + nodeInfo, err = handshake(c, mt.handshakeTimeout, mt.nodeInfo) + if err != nil { + return nil, nil, ErrRejected{ + conn: c, + err: fmt.Errorf("handshake failed: %v", err), + isAuthFailure: true, + } + } + + if err := nodeInfo.Validate(); err != nil { + return nil, nil, ErrRejected{ + conn: c, + err: err, + isNodeInfoInvalid: true, + } + } + + // Ensure connection key matches self reported key. + //if connID != nodeInfo.ID() { + // return nil, nil, ErrRejected{ + // conn: c, + // id: connID, + // err: fmt.Errorf( + // "conn.ID (%v) NodeInfo.ID (%v) mismatch", + // connID, + // nodeInfo.ID(), + // ), + // isAuthFailure: true, + // } + //} + + // Reject self. + if mt.nodeInfo.ID() == nodeInfo.ID() { + return nil, nil, ErrRejected{ + addr: *NewNetAddress(nodeInfo.ID(), c.RemoteAddr()), + conn: c, + id: nodeInfo.ID(), + isSelf: true, + } + } + + if err := mt.nodeInfo.CompatibleWith(nodeInfo); err != nil { + return nil, nil, ErrRejected{ + conn: c, + err: err, + id: nodeInfo.ID(), + isIncompatible: true, + } + } + return c, nodeInfo, nil } +func handshake( + c quic.Connection, + timeout time.Duration, + nodeInfo NodeInfo, +) (NodeInfo, error) { + var ( + errc = make(chan error, 2) + + pbpeerNodeInfo tmp2p.DefaultNodeInfo + peerNodeInfo DefaultNodeInfo + ourNodeInfo = nodeInfo.(DefaultNodeInfo) + ) + + go func(errc chan<- error, c quic.Connection) { + stream, err := c.OpenStream() + if err != nil { + errc <- err + return + } + _, err = protoio.NewDelimitedWriter(stream).WriteMsg(ourNodeInfo.ToProto()) + errc <- err + }(errc, c) + go func(errc chan<- error, c quic.Connection) { + stream, err := c.AcceptStream(context.Background()) + if err != nil { + errc <- err + return + } + protoReader := protoio.NewDelimitedReader(stream, MaxNodeInfoSize()) + _, err = protoReader.ReadMsg(&pbpeerNodeInfo) + errc <- err + }(errc, c) + + for i := 0; i < cap(errc); i++ { + err := <-errc + if err != nil { + return nil, err + } + } + + peerNodeInfo, err := DefaultNodeInfoFromToProto(&pbpeerNodeInfo) + if err != nil { + return nil, err + } + + return peerNodeInfo, nil +} + func (mt *MultiplexTransport) wrapPeer( c quic.Connection, ni NodeInfo, From 6851c78e0afa1ed5c41dfa96d51039a30213f39a Mon Sep 17 00:00:00 2001 From: sweexordious Date: Wed, 14 Aug 2024 08:39:48 +0100 Subject: [PATCH 03/55] connected but not sending/receiving --- p2p/peer.go | 163 ++++++++++++++++++++++++++--------------------- p2p/test_util.go | 11 +--- p2p/transport.go | 2 + 3 files changed, 95 insertions(+), 81 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index 197f6e683a..9530901dd8 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -1,9 +1,13 @@ package p2p import ( + "context" + "errors" "fmt" "github.com/quic-go/quic-go" + "github.com/tendermint/tendermint/pkg/trace/schema" "net" + "reflect" "time" "github.com/gogo/protobuf/proto" @@ -163,6 +167,8 @@ type peer struct { // raw peerConn and the multiplex connection peerConn + onReceive func(chID byte, msgBytes []byte) + // peer's node info and the channel it knows about // channels = nodeInfo.Channels // cached to avoid copying nodeInfo in hasChannel @@ -194,6 +200,8 @@ func WithPeerTracer(t trace.Tracer) PeerOption { func newPeer( pc peerConn, nodeInfo NodeInfo, + reactorsByCh map[byte]Reactor, + msgTypeByChID map[byte]proto.Message, mlc *metricsLabelCache, options ...PeerOption, ) *peer { @@ -209,15 +217,52 @@ func newPeer( streams: make(map[byte]quic.Stream), } - //p.mconn = createMConnection( - // pc.conn, - // p, - // reactorsByCh, - // msgTypeByChID, - // chDescs, - // onPeerError, - // mConfig, - //) + p.onReceive = func(chID byte, msgBytes []byte) { + reactor := reactorsByCh[chID] + if reactor == nil { + // Note that its ok to panic here as it's caught in the conn._recover, + // which does onPeerError. + panic(fmt.Sprintf("Unknown channel %X", chID)) + } + mt := msgTypeByChID[chID] + msg := proto.Clone(mt) + err := proto.Unmarshal(msgBytes, msg) + if err != nil { + panic(fmt.Errorf("unmarshaling message: %s into type: %s", err, reflect.TypeOf(mt))) + } + + if w, ok := msg.(Unwrapper); ok { + msg, err = w.Unwrap() + if err != nil { + panic(fmt.Errorf("unwrapping message: %s", err)) + } + } + + labels := []string{ + "peer_id", string(p.ID()), + "chID", fmt.Sprintf("%#x", chID), + } + + p.metrics.PeerReceiveBytesTotal.With(labels...).Add(float64(len(msgBytes))) + p.metrics.MessageReceiveBytesTotal.With(append(labels, "message_type", p.mlc.ValueToMetricLabel(msg))...).Add(float64(len(msgBytes))) + schema.WriteReceivedBytes(p.traceClient, string(p.ID()), chID, len(msgBytes)) + if nr, ok := reactor.(EnvelopeReceiver); ok { + nr.ReceiveEnvelope(Envelope{ + ChannelID: chID, + Src: p, + Message: msg, + }) + } else { + reactor.Receive(chID, p, msgBytes) + } + } + + go func() { + err := p.OnReceive() + if err != nil { + p.Logger.Error("error receiving stuff", "err", err.Error()) + } + }() p.BaseService = *service.NewBaseService(nil, "Peer", p) for _, option := range options { @@ -362,10 +407,18 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { } p.streams[chID] = newStream stream = newStream + _, err = stream.Write([]byte{chID}) + if err != nil { + p.Logger.Error("error sending channel ID", "err", err.Error()) + return false + } + p.Logger.Info("successfully shared channel ID", "id", chID) } + p.Logger.Debug("Send", "channel", chID, "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) n, err := stream.Write(msgBytes) if err != nil { + p.Logger.Debug("Send failed", "channel", "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) return false } labels := []string{ @@ -480,65 +533,33 @@ func (p *peer) metricsReporter() { } } -//func createMConnection( -// conn quic.Connection, -// p *peer, -// reactorsByCh map[byte]Reactor, -// msgTypeByChID map[byte]proto.Message, -// chDescs []*cmtconn.ChannelDescriptor, -// onPeerError func(Peer, interface{}), -// config cmtconn.MConnConfig, -//) *cmtconn.MConnection { -// -// onReceive := func(chID byte, msgBytes []byte) { -// reactor := reactorsByCh[chID] -// if reactor == nil { -// // Note that its ok to panic here as it's caught in the conn._recover, -// // which does onPeerError. -// panic(fmt.Sprintf("Unknown channel %X", chID)) -// } -// mt := msgTypeByChID[chID] -// msg := proto.Clone(mt) -// err := proto.Unmarshal(msgBytes, msg) -// if err != nil { -// panic(fmt.Errorf("unmarshaling message: %s into type: %s", err, reflect.TypeOf(mt))) -// } -// -// if w, ok := msg.(Unwrapper); ok { -// msg, err = w.Unwrap() -// if err != nil { -// panic(fmt.Errorf("unwrapping message: %s", err)) -// } -// } -// -// labels := []string{ -// "peer_id", string(p.ID()), -// "chID", fmt.Sprintf("%#x", chID), -// } -// -// p.metrics.PeerReceiveBytesTotal.With(labels...).Add(float64(len(msgBytes))) -// p.metrics.MessageReceiveBytesTotal.With(append(labels, "message_type", p.mlc.ValueToMetricLabel(msg))...).Add(float64(len(msgBytes))) -// schema.WriteReceivedBytes(p.traceClient, string(p.ID()), chID, len(msgBytes)) -// if nr, ok := reactor.(EnvelopeReceiver); ok { -// nr.ReceiveEnvelope(Envelope{ -// ChannelID: chID, -// Src: p, -// Message: msg, -// }) -// } else { -// reactor.Receive(chID, p, msgBytes) -// } -// } -// -// onError := func(r interface{}) { -// onPeerError(p, r) -// } -// -// return cmtconn.NewMConnectionWithConfig( -// conn, -// chDescs, -// onReceive, -// onError, -// config, -// ) -//} +func (p *peer) OnReceive() error { + for i := 0; i < 500; i++ { + stream, err := p.conn.AcceptStream(context.Background()) + if err != nil { + p.Logger.Error("failed to accept stream", "err", err.Error()) + return err + } + go func() { + chID := make([]byte, 1) + _, err := stream.Read(chID) + if err != nil { + p.Logger.Error("failed to read channel ID", "err", err.Error()) + return + } + p.streams[chID[0]] = stream + p.Logger.Info("successfully got channel ID", "id", chID[0]) + // start accepting data + go func() { + data := make([]byte, 1000000000) + n, err := stream.Read(data) + if err != nil { + p.Logger.Error("failed to read from stream", "err", err.Error()) + return + } + p.onReceive(chID[0], data[:n]) + }() + }() + } + return errors.New("reached 500 opened streams with peer") +} diff --git a/p2p/test_util.go b/p2p/test_util.go index 8c41148631..147aa89ef8 100644 --- a/p2p/test_util.go +++ b/p2p/test_util.go @@ -152,16 +152,7 @@ func (sw *Switch) addPeerWithConnection(conn net.Conn) error { return err } - p := newPeer( - pc, - MConnConfig(sw.config), - ni, - sw.reactorsByCh, - sw.msgTypeByChID, - sw.chDescs, - sw.StopPeerForError, - sw.mlc, - ) + p := newPeer(pc, ni, sw.reactorsByCh, sw.msgTypeByChID, sw.mlc) if err = sw.addPeer(p); err != nil { pc.CloseConn() diff --git a/p2p/transport.go b/p2p/transport.go index 6b7cec76e1..0b15012693 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -588,6 +588,8 @@ func (mt *MultiplexTransport) wrapPeer( p := newPeer( peerConn, ni, + cfg.reactorsByCh, + cfg.msgTypeByChID, cfg.mlc, PeerMetrics(cfg.metrics), WithPeerTracer(mt.tracer), From 74930f05cf4f670cd748b3e212f2a71cf9c695c3 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Fri, 16 Aug 2024 01:08:43 +0100 Subject: [PATCH 04/55] working connection and some messages pass but still slow and panicing --- node/node.go | 3 +- p2p/conn/connection.go | 822 ------------------------------------ p2p/netaddress.go | 16 +- p2p/peer.go | 87 ++-- p2p/switch.go | 11 - p2p/transport.go | 9 +- p2p/types.go | 1 - rpc/core/types/responses.go | 7 +- 8 files changed, 77 insertions(+), 879 deletions(-) diff --git a/node/node.go b/node/node.go index 1827609db7..6363b4e704 100644 --- a/node/node.go +++ b/node/node.go @@ -556,8 +556,7 @@ func createTransport( []p2p.PeerFilterFunc, ) { var ( - mConnConfig = p2p.MConnConfig(config.P2P) - transport = p2p.NewMultiplexTransport(nodeInfo, *nodeKey, mConnConfig, tracer) + transport = p2p.NewMultiplexTransport(nodeInfo, *nodeKey, tracer) connFilters = []p2p.ConnFilterFunc{} peerFilters = []p2p.PeerFilterFunc{} ) diff --git a/p2p/conn/connection.go b/p2p/conn/connection.go index b09e29be5b..6e166983ca 100644 --- a/p2p/conn/connection.go +++ b/p2p/conn/connection.go @@ -1,26 +1,11 @@ package conn import ( - "bufio" - "errors" "fmt" - "io" - "math" - "net" - "reflect" - "runtime/debug" - "sync/atomic" "time" "github.com/gogo/protobuf/proto" - flow "github.com/tendermint/tendermint/libs/flowrate" - "github.com/tendermint/tendermint/libs/log" - cmtmath "github.com/tendermint/tendermint/libs/math" - "github.com/tendermint/tendermint/libs/protoio" - "github.com/tendermint/tendermint/libs/service" - cmtsync "github.com/tendermint/tendermint/libs/sync" - "github.com/tendermint/tendermint/libs/timer" tmp2p "github.com/tendermint/tendermint/proto/tendermint/p2p" ) @@ -50,672 +35,6 @@ const ( type receiveCbFunc func(chID byte, msgBytes []byte) type errorCbFunc func(interface{}) -/* -Each peer has one `MConnection` (multiplex connection) instance. - -__multiplex__ *noun* a system or signal involving simultaneous transmission of -several messages along a single channel of communication. - -Each `MConnection` handles message transmission on multiple abstract communication -`Channel`s. Each channel has a globally unique byte id. -The byte id and the relative priorities of each `Channel` are configured upon -initialization of the connection. - -There are two methods for sending messages: - - func (m MConnection) Send(chID byte, msgBytes []byte) bool {} - func (m MConnection) TrySend(chID byte, msgBytes []byte}) bool {} - -`Send(chID, msgBytes)` is a blocking call that waits until `msg` is -successfully queued for the channel with the given id byte `chID`, or until the -request times out. The message `msg` is serialized using Protobuf. - -`TrySend(chID, msgBytes)` is a nonblocking call that returns false if the -channel's queue is full. - -Inbound message bytes are handled with an onReceive callback function. -*/ -type MConnection struct { - service.BaseService - - conn net.Conn - bufConnReader *bufio.Reader - bufConnWriter *bufio.Writer - sendMonitor *flow.Monitor - recvMonitor *flow.Monitor - send chan struct{} - pong chan struct{} - channels []*Channel - channelsIdx map[byte]*Channel - onReceive receiveCbFunc - onError errorCbFunc - errored uint32 - config MConnConfig - - // Closing quitSendRoutine will cause the sendRoutine to eventually quit. - // doneSendRoutine is closed when the sendRoutine actually quits. - quitSendRoutine chan struct{} - doneSendRoutine chan struct{} - - // Closing quitRecvRouting will cause the recvRouting to eventually quit. - quitRecvRoutine chan struct{} - - // used to ensure FlushStop and OnStop - // are safe to call concurrently. - stopMtx cmtsync.Mutex - - flushTimer *timer.ThrottleTimer // flush writes as necessary but throttled. - pingTimer *time.Ticker // send pings periodically - - // close conn if pong is not received in pongTimeout - pongTimer *time.Timer - pongTimeoutCh chan bool // true - timeout, false - peer sent pong - - chStatsTimer *time.Ticker // update channel stats periodically - - created time.Time // time of creation - - _maxPacketMsgSize int -} - -// MConnConfig is a MConnection configuration. -type MConnConfig struct { - SendRate int64 `mapstructure:"send_rate"` - RecvRate int64 `mapstructure:"recv_rate"` - - // Maximum payload size - MaxPacketMsgPayloadSize int `mapstructure:"max_packet_msg_payload_size"` - - // Interval to flush writes (throttled) - FlushThrottle time.Duration `mapstructure:"flush_throttle"` - - // Interval to send pings - PingInterval time.Duration `mapstructure:"ping_interval"` - - // Maximum wait time for pongs - PongTimeout time.Duration `mapstructure:"pong_timeout"` -} - -// DefaultMConnConfig returns the default config. -func DefaultMConnConfig() MConnConfig { - return MConnConfig{ - SendRate: defaultSendRate, - RecvRate: defaultRecvRate, - MaxPacketMsgPayloadSize: defaultMaxPacketMsgPayloadSize, - FlushThrottle: defaultFlushThrottle, - PingInterval: defaultPingInterval, - PongTimeout: defaultPongTimeout, - } -} - -// NewMConnection wraps net.Conn and creates multiplex connection -func NewMConnection( - conn net.Conn, - chDescs []*ChannelDescriptor, - onReceive receiveCbFunc, - onError errorCbFunc, -) *MConnection { - return NewMConnectionWithConfig( - conn, - chDescs, - onReceive, - onError, - DefaultMConnConfig()) -} - -// NewMConnectionWithConfig wraps net.Conn and creates multiplex connection with a config -func NewMConnectionWithConfig( - conn net.Conn, - chDescs []*ChannelDescriptor, - onReceive receiveCbFunc, - onError errorCbFunc, - config MConnConfig, -) *MConnection { - if config.PongTimeout >= config.PingInterval { - panic("pongTimeout must be less than pingInterval (otherwise, next ping will reset pong timer)") - } - - mconn := &MConnection{ - conn: conn, - bufConnReader: bufio.NewReaderSize(conn, minReadBufferSize), - bufConnWriter: bufio.NewWriterSize(conn, minWriteBufferSize), - sendMonitor: flow.New(0, 0), - recvMonitor: flow.New(0, 0), - send: make(chan struct{}, 1), - pong: make(chan struct{}, 1), - onReceive: onReceive, - onError: onError, - config: config, - created: time.Now(), - } - - // Create channels - var channelsIdx = map[byte]*Channel{} - var channels = []*Channel{} - - for _, desc := range chDescs { - channel := newChannel(mconn, *desc) - channelsIdx[channel.desc.ID] = channel - channels = append(channels, channel) - } - mconn.channels = channels - mconn.channelsIdx = channelsIdx - - mconn.BaseService = *service.NewBaseService(nil, "MConnection", mconn) - - // maxPacketMsgSize() is a bit heavy, so call just once - mconn._maxPacketMsgSize = mconn.maxPacketMsgSize() - - return mconn -} - -func (c *MConnection) SetLogger(l log.Logger) { - c.BaseService.SetLogger(l) - for _, ch := range c.channels { - ch.SetLogger(l) - } -} - -// OnStart implements BaseService -func (c *MConnection) OnStart() error { - if err := c.BaseService.OnStart(); err != nil { - return err - } - c.flushTimer = timer.NewThrottleTimer("flush", c.config.FlushThrottle) - c.pingTimer = time.NewTicker(c.config.PingInterval) - c.pongTimeoutCh = make(chan bool, 1) - c.chStatsTimer = time.NewTicker(updateStats) - c.quitSendRoutine = make(chan struct{}) - c.doneSendRoutine = make(chan struct{}) - c.quitRecvRoutine = make(chan struct{}) - go c.sendRoutine() - go c.recvRoutine() - return nil -} - -// stopServices stops the BaseService and timers and closes the quitSendRoutine. -// if the quitSendRoutine was already closed, it returns true, otherwise it returns false. -// It uses the stopMtx to ensure only one of FlushStop and OnStop can do this at a time. -func (c *MConnection) stopServices() (alreadyStopped bool) { - c.stopMtx.Lock() - defer c.stopMtx.Unlock() - - select { - case <-c.quitSendRoutine: - // already quit - return true - default: - } - - select { - case <-c.quitRecvRoutine: - // already quit - return true - default: - } - - c.BaseService.OnStop() - c.flushTimer.Stop() - c.pingTimer.Stop() - c.chStatsTimer.Stop() - - // inform the recvRouting that we are shutting down - close(c.quitRecvRoutine) - close(c.quitSendRoutine) - return false -} - -// FlushStop replicates the logic of OnStop. -// It additionally ensures that all successful -// .Send() calls will get flushed before closing -// the connection. -func (c *MConnection) FlushStop() { - if c.stopServices() { - return - } - - // this block is unique to FlushStop - { - // wait until the sendRoutine exits - // so we dont race on calling sendSomePacketMsgs - <-c.doneSendRoutine - - // Send and flush all pending msgs. - // Since sendRoutine has exited, we can call this - // safely - eof := c.sendSomePacketMsgs() - for !eof { - eof = c.sendSomePacketMsgs() - } - c.flush() - - // Now we can close the connection - } - - c.conn.Close() - - // We can't close pong safely here because - // recvRoutine may write to it after we've stopped. - // Though it doesn't need to get closed at all, - // we close it @ recvRoutine. - - // c.Stop() -} - -// OnStop implements BaseService -func (c *MConnection) OnStop() { - if c.stopServices() { - return - } - - c.conn.Close() - - // We can't close pong safely here because - // recvRoutine may write to it after we've stopped. - // Though it doesn't need to get closed at all, - // we close it @ recvRoutine. -} - -func (c *MConnection) String() string { - return fmt.Sprintf("MConn{%v}", c.conn.RemoteAddr()) -} - -func (c *MConnection) flush() { - c.Logger.Debug("Flush", "conn", c) - err := c.bufConnWriter.Flush() - if err != nil { - c.Logger.Debug("MConnection flush failed", "err", err) - } -} - -// Catch panics, usually caused by remote disconnects. -func (c *MConnection) _recover() { - if r := recover(); r != nil { - c.Logger.Error("MConnection panicked", "err", r, "stack", string(debug.Stack())) - c.stopForError(fmt.Errorf("recovered from panic: %v", r)) - } -} - -func (c *MConnection) stopForError(r interface{}) { - if err := c.Stop(); err != nil { - c.Logger.Error("Error stopping connection", "err", err) - } - if atomic.CompareAndSwapUint32(&c.errored, 0, 1) { - if c.onError != nil { - c.onError(r) - } - } -} - -// Queues a message to be sent to channel. -func (c *MConnection) Send(chID byte, msgBytes []byte) bool { - if !c.IsRunning() { - return false - } - - c.Logger.Debug("Send", "channel", chID, "conn", c, "msgBytes", log.NewLazySprintf("%X", msgBytes)) - - // Send message to channel. - channel, ok := c.channelsIdx[chID] - if !ok { - c.Logger.Error(fmt.Sprintf("Cannot send bytes, unknown channel %X", chID)) - return false - } - - success := channel.sendBytes(msgBytes) - if success { - // Wake up sendRoutine if necessary - select { - case c.send <- struct{}{}: - default: - } - } else { - c.Logger.Debug("Send failed", "channel", chID, "conn", c, "msgBytes", log.NewLazySprintf("%X", msgBytes)) - } - return success -} - -// Queues a message to be sent to channel. -// Nonblocking, returns true if successful. -func (c *MConnection) TrySend(chID byte, msgBytes []byte) bool { - if !c.IsRunning() { - return false - } - - c.Logger.Debug("TrySend", "channel", chID, "conn", c, "msgBytes", log.NewLazySprintf("%X", msgBytes)) - - // Send message to channel. - channel, ok := c.channelsIdx[chID] - if !ok { - c.Logger.Error(fmt.Sprintf("Cannot send bytes, unknown channel %X", chID)) - return false - } - - ok = channel.trySendBytes(msgBytes) - if ok { - // Wake up sendRoutine if necessary - select { - case c.send <- struct{}{}: - default: - } - } - - return ok -} - -// CanSend returns true if you can send more data onto the chID, false -// otherwise. Use only as a heuristic. -func (c *MConnection) CanSend(chID byte) bool { - if !c.IsRunning() { - return false - } - - channel, ok := c.channelsIdx[chID] - if !ok { - c.Logger.Error(fmt.Sprintf("Unknown channel %X", chID)) - return false - } - return channel.canSend() -} - -// sendRoutine polls for packets to send from channels. -func (c *MConnection) sendRoutine() { - defer c._recover() - - protoWriter := protoio.NewDelimitedWriter(c.bufConnWriter) - -FOR_LOOP: - for { - //var _n int - var err error - SELECTION: - select { - case <-c.flushTimer.Ch: - // NOTE: flushTimer.Set() must be called every time - // something is written to .bufConnWriter. - c.flush() - case <-c.chStatsTimer.C: - for _, channel := range c.channels { - channel.updateStats() - } - case <-c.pingTimer.C: - c.Logger.Debug("Send Ping") - _, err = protoWriter.WriteMsg(mustWrapPacket(&tmp2p.PacketPing{})) - if err != nil { - c.Logger.Error("Failed to send PacketPing", "err", err) - break SELECTION - } - //c.sendMonitor.Update(_n) - c.Logger.Debug("Starting pong timer", "dur", c.config.PongTimeout) - c.pongTimer = time.AfterFunc(c.config.PongTimeout, func() { - select { - case c.pongTimeoutCh <- true: - default: - } - }) - c.flush() - case timeout := <-c.pongTimeoutCh: - if timeout { - c.Logger.Debug("Pong timeout") - err = errors.New("pong timeout") - } else { - c.stopPongTimer() - } - case <-c.pong: - c.Logger.Debug("Send Pong") - _, err = protoWriter.WriteMsg(mustWrapPacket(&tmp2p.PacketPong{})) - if err != nil { - c.Logger.Error("Failed to send PacketPong", "err", err) - break SELECTION - } - //c.sendMonitor.Update(_n) - c.flush() - case <-c.quitSendRoutine: - break FOR_LOOP - case <-c.send: - // Send some PacketMsgs - eof := c.sendSomePacketMsgs() - if !eof { - // Keep sendRoutine awake. - select { - case c.send <- struct{}{}: - default: - } - } - } - - if !c.IsRunning() { - break FOR_LOOP - } - if err != nil { - c.Logger.Error("Connection failed @ sendRoutine", "conn", c, "err", err) - c.stopForError(err) - break FOR_LOOP - } - } - - // Cleanup - c.stopPongTimer() - close(c.doneSendRoutine) -} - -// Returns true if messages from channels were exhausted. -// Blocks in accordance to .sendMonitor throttling. -func (c *MConnection) sendSomePacketMsgs() bool { - // Block until .sendMonitor says we can write. - // Once we're ready we send more than we asked for, - // but amortized it should even out. - //c.sendMonitor.Limit(c._maxPacketMsgSize, atomic.LoadInt64(&c.config.SendRate), true) - - // Now send some PacketMsgs. - for i := 0; i < numBatchPacketMsgs; i++ { - if c.sendPacketMsg() { - return true - } - } - return false -} - -// Returns true if messages from channels were exhausted. -func (c *MConnection) sendPacketMsg() bool { - // Choose a channel to create a PacketMsg from. - // The chosen channel will be the one whose recentlySent/priority is the least. - var leastRatio float32 = math.MaxFloat32 - var leastChannel *Channel - for _, channel := range c.channels { - // If nothing to send, skip this channel - if !channel.isSendPending() { - continue - } - // Get ratio, and keep track of lowest ratio. - ratio := float32(channel.recentlySent) / float32(channel.desc.Priority) - if ratio < leastRatio { - leastRatio = ratio - leastChannel = channel - } - } - - // Nothing to send? - if leastChannel == nil { - return true - } - // c.Logger.Info("Found a msgPacket to send") - - // Make & send a PacketMsg from this channel - _, err := leastChannel.writePacketMsgTo(c.bufConnWriter) - if err != nil { - c.Logger.Error("Failed to write PacketMsg", "err", err) - c.stopForError(err) - return true - } - //c.sendMonitor.Update(_n) - c.flushTimer.Set() - return false -} - -// recvRoutine reads PacketMsgs and reconstructs the message using the channels' "recving" buffer. -// After a whole message has been assembled, it's pushed to onReceive(). -// Blocks depending on how the connection is throttled. -// Otherwise, it never blocks. -func (c *MConnection) recvRoutine() { - defer c._recover() - - protoReader := protoio.NewDelimitedReader(c.bufConnReader, c._maxPacketMsgSize) - -FOR_LOOP: - for { - // Block until .recvMonitor says we can read. - //c.recvMonitor.Limit(c._maxPacketMsgSize, atomic.LoadInt64(&c.config.RecvRate), true) - - // Peek into bufConnReader for debugging - /* - if numBytes := c.bufConnReader.Buffered(); numBytes > 0 { - bz, err := c.bufConnReader.Peek(cmtmath.MinInt(numBytes, 100)) - if err == nil { - // return - } else { - c.Logger.Debug("Error peeking connection buffer", "err", err) - // return nil - } - c.Logger.Info("Peek connection buffer", "numBytes", numBytes, "bz", bz) - } - */ - - // Read packet type - var packet tmp2p.Packet - - _, err := protoReader.ReadMsg(&packet) - //c.recvMonitor.Update(_n) - if err != nil { - // stopServices was invoked and we are shutting down - // receiving is excpected to fail since we will close the connection - select { - case <-c.quitRecvRoutine: - break FOR_LOOP - default: - } - - if c.IsRunning() { - if err == io.EOF { - c.Logger.Info("Connection is closed @ recvRoutine (likely by the other side)", "conn", c) - } else { - c.Logger.Debug("Connection failed @ recvRoutine (reading byte)", "conn", c, "err", err) - } - c.stopForError(err) - } - break FOR_LOOP - } - - // Read more depending on packet type. - switch pkt := packet.Sum.(type) { - case *tmp2p.Packet_PacketPing: - // TODO: prevent abuse, as they cause flush()'s. - // https://github.com/tendermint/tendermint/issues/1190 - c.Logger.Debug("Receive Ping") - select { - case c.pong <- struct{}{}: - default: - // never block - } - case *tmp2p.Packet_PacketPong: - c.Logger.Debug("Receive Pong") - select { - case c.pongTimeoutCh <- false: - default: - // never block - } - case *tmp2p.Packet_PacketMsg: - channelID := byte(pkt.PacketMsg.ChannelID) - channel, ok := c.channelsIdx[channelID] - if pkt.PacketMsg.ChannelID < 0 || pkt.PacketMsg.ChannelID > math.MaxUint8 || !ok || channel == nil { - err := fmt.Errorf("unknown channel %X", pkt.PacketMsg.ChannelID) - c.Logger.Debug("Connection failed @ recvRoutine", "conn", c, "err", err) - c.stopForError(err) - break FOR_LOOP - } - - msgBytes, err := channel.recvPacketMsg(*pkt.PacketMsg) - if err != nil { - if c.IsRunning() { - c.Logger.Debug("Connection failed @ recvRoutine", "conn", c, "err", err) - c.stopForError(err) - } - break FOR_LOOP - } - if msgBytes != nil { - c.Logger.Debug("Received bytes", "chID", channelID, "msgBytes", msgBytes) - // NOTE: This means the reactor.Receive runs in the same thread as the p2p recv routine - c.onReceive(channelID, msgBytes) - } - default: - err := fmt.Errorf("unknown message type %v", reflect.TypeOf(packet)) - c.Logger.Error("Connection failed @ recvRoutine", "conn", c, "err", err) - c.stopForError(err) - break FOR_LOOP - } - } - - // Cleanup - close(c.pong) - for range c.pong { - // Drain - } -} - -// not goroutine-safe -func (c *MConnection) stopPongTimer() { - if c.pongTimer != nil { - _ = c.pongTimer.Stop() - c.pongTimer = nil - } -} - -// maxPacketMsgSize returns a maximum size of PacketMsg -func (c *MConnection) maxPacketMsgSize() int { - bz, err := proto.Marshal(mustWrapPacket(&tmp2p.PacketMsg{ - ChannelID: 0x01, - EOF: true, - Data: make([]byte, c.config.MaxPacketMsgPayloadSize), - })) - if err != nil { - panic(err) - } - return len(bz) -} - -type ConnectionStatus struct { - Duration time.Duration - SendMonitor flow.Status - RecvMonitor flow.Status - Channels []ChannelStatus -} - -type ChannelStatus struct { - ID byte - SendQueueCapacity int - SendQueueSize int - Priority int - RecentlySent int64 -} - -func (c *MConnection) Status() ConnectionStatus { - var status ConnectionStatus - status.Duration = time.Since(c.created) - //status.SendMonitor = c.sendMonitor.Status() - //status.RecvMonitor = c.recvMonitor.Status() - status.Channels = make([]ChannelStatus, len(c.channels)) - for i, channel := range c.channels { - status.Channels[i] = ChannelStatus{ - ID: channel.desc.ID, - SendQueueCapacity: cap(channel.sendQueue), - SendQueueSize: int(atomic.LoadInt32(&channel.sendQueueSize)), - Priority: channel.desc.Priority, - RecentlySent: atomic.LoadInt64(&channel.recentlySent), - } - } - return status -} - //----------------------------------------------------------------------------- type ChannelDescriptor struct { @@ -741,147 +60,6 @@ func (chDesc ChannelDescriptor) FillDefaults() (filled ChannelDescriptor) { return } -// TODO: lowercase. -// NOTE: not goroutine-safe. -type Channel struct { - conn *MConnection - desc ChannelDescriptor - sendQueue chan []byte - sendQueueSize int32 // atomic. - recving []byte - sending []byte - recentlySent int64 // exponential moving average - - maxPacketMsgPayloadSize int - - Logger log.Logger -} - -func newChannel(conn *MConnection, desc ChannelDescriptor) *Channel { - desc = desc.FillDefaults() - if desc.Priority <= 0 { - panic("Channel default priority must be a positive integer") - } - return &Channel{ - conn: conn, - desc: desc, - sendQueue: make(chan []byte, desc.SendQueueCapacity), - recving: make([]byte, 0, desc.RecvBufferCapacity), - maxPacketMsgPayloadSize: conn.config.MaxPacketMsgPayloadSize, - } -} - -func (ch *Channel) SetLogger(l log.Logger) { - ch.Logger = l -} - -// Queues message to send to this channel. -// Goroutine-safe -// Times out (and returns false) after defaultSendTimeout -func (ch *Channel) sendBytes(bytes []byte) bool { - select { - case ch.sendQueue <- bytes: - atomic.AddInt32(&ch.sendQueueSize, 1) - return true - case <-time.After(defaultSendTimeout): - return false - } -} - -// Queues message to send to this channel. -// Nonblocking, returns true if successful. -// Goroutine-safe -func (ch *Channel) trySendBytes(bytes []byte) bool { - select { - case ch.sendQueue <- bytes: - atomic.AddInt32(&ch.sendQueueSize, 1) - return true - default: - return false - } -} - -// Goroutine-safe -func (ch *Channel) loadSendQueueSize() (size int) { - return int(atomic.LoadInt32(&ch.sendQueueSize)) -} - -// Goroutine-safe -// Use only as a heuristic. -func (ch *Channel) canSend() bool { - return ch.loadSendQueueSize() < defaultSendQueueCapacity -} - -// Returns true if any PacketMsgs are pending to be sent. -// Call before calling nextPacketMsg() -// Goroutine-safe -func (ch *Channel) isSendPending() bool { - if len(ch.sending) == 0 { - if len(ch.sendQueue) == 0 { - return false - } - ch.sending = <-ch.sendQueue - } - return true -} - -// Creates a new PacketMsg to send. -// Not goroutine-safe -func (ch *Channel) nextPacketMsg() tmp2p.PacketMsg { - packet := tmp2p.PacketMsg{ChannelID: int32(ch.desc.ID)} - maxSize := ch.maxPacketMsgPayloadSize - packet.Data = ch.sending[:cmtmath.MinInt(maxSize, len(ch.sending))] - if len(ch.sending) <= maxSize { - packet.EOF = true - ch.sending = nil - atomic.AddInt32(&ch.sendQueueSize, -1) // decrement sendQueueSize - } else { - packet.EOF = false - ch.sending = ch.sending[cmtmath.MinInt(maxSize, len(ch.sending)):] - } - return packet -} - -// Writes next PacketMsg to w and updates c.recentlySent. -// Not goroutine-safe -func (ch *Channel) writePacketMsgTo(w io.Writer) (n int, err error) { - packet := ch.nextPacketMsg() - n, err = protoio.NewDelimitedWriter(w).WriteMsg(mustWrapPacket(&packet)) - atomic.AddInt64(&ch.recentlySent, int64(n)) - return -} - -// Handles incoming PacketMsgs. It returns a message bytes if message is -// complete. NOTE message bytes may change on next call to recvPacketMsg. -// Not goroutine-safe -func (ch *Channel) recvPacketMsg(packet tmp2p.PacketMsg) ([]byte, error) { - ch.Logger.Debug("Read PacketMsg", "conn", ch.conn, "packet", packet) - //var recvCap, recvReceived = ch.desc.RecvMessageCapacity, len(ch.recving) + len(packet.Data) - //if recvCap < recvReceived { - // return nil, fmt.Errorf("received message exceeds available capacity: %v < %v", recvCap, recvReceived) - //} - ch.recving = append(ch.recving, packet.Data...) - if packet.EOF { - msgBytes := ch.recving - - // clear the slice without re-allocating. - // http://stackoverflow.com/questions/16971741/how-do-you-clear-a-slice-in-go - // suggests this could be a memory leak, but we might as well keep the memory for the channel until it closes, - // at which point the recving slice stops being used and should be garbage collected - ch.recving = ch.recving[:0] // make([]byte, 0, ch.desc.RecvBufferCapacity) - return msgBytes, nil - } - return nil, nil -} - -// Call this periodically to update stats for throttling purposes. -// Not goroutine-safe -func (ch *Channel) updateStats() { - // Exponential decay of stats. - // TODO: optimize. - atomic.StoreInt64(&ch.recentlySent, int64(float64(atomic.LoadInt64(&ch.recentlySent))*0.8)) -} - //---------------------------------------- // Packet diff --git a/p2p/netaddress.go b/p2p/netaddress.go index 4ade8bf65e..12da0bfab2 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -240,7 +240,13 @@ func (na *NetAddress) Dial() (quic.Connection, error) { tlsConfig := tls.Config{ InsecureSkipVerify: true, } - quickConfig := quic.Config{} + quickConfig := quic.Config{ + MaxIdleTimeout: 10 * time.Minute, + MaxIncomingStreams: 1000000000, + MaxIncomingUniStreams: 1000000000, + KeepAlivePeriod: 100 * time.Millisecond, + EnableDatagrams: true, + } conn, err := quic.DialAddr(context.Background(), na.DialString(), &tlsConfig, &quickConfig) if err != nil { return nil, err @@ -253,7 +259,13 @@ func (na *NetAddress) DialTimeout(timeout time.Duration) (quic.Connection, error tlsConfig := tls.Config{ InsecureSkipVerify: true, } - quickConfig := quic.Config{} + quickConfig := quic.Config{ + MaxIdleTimeout: 10 * time.Minute, + MaxIncomingStreams: 1000000000, + MaxIncomingUniStreams: 1000000000, + KeepAlivePeriod: 100 * time.Millisecond, + EnableDatagrams: true, + } conn, err := quic.DialAddr(context.Background(), na.DialString(), &tlsConfig, &quickConfig) if err != nil { return nil, err diff --git a/p2p/peer.go b/p2p/peer.go index 9530901dd8..6393fcce58 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -2,10 +2,13 @@ package p2p import ( "context" + "encoding/binary" + "encoding/hex" "errors" "fmt" "github.com/quic-go/quic-go" "github.com/tendermint/tendermint/pkg/trace/schema" + "io" "net" "reflect" "time" @@ -66,6 +69,7 @@ type EnvelopeSender interface { // // Deprecated: Will be removed in v0.37. func SendEnvelopeShim(p Peer, e Envelope, lg log.Logger) bool { + fmt.Println(e) if es, ok := p.(EnvelopeSender); ok { return es.SendEnvelope(e) } @@ -218,6 +222,7 @@ func newPeer( } p.onReceive = func(chID byte, msgBytes []byte) { + //p.Logger.Info("handling data", "channel", chID, "data", hex.EncodeToString(msgBytes)) reactor := reactorsByCh[chID] if reactor == nil { // Note that its ok to panic here as it's caught in the conn._recover, @@ -228,9 +233,11 @@ func newPeer( msg := proto.Clone(mt) err := proto.Unmarshal(msgBytes, msg) if err != nil { + p.Logger.Error("before panic", "msg", msg, "type", mt, "bytes", hex.EncodeToString(msgBytes)) panic(fmt.Errorf("unmarshaling message: %s into type: %s", err, reflect.TypeOf(mt))) } + p.Logger.Error("type of message", "type", msg) if w, ok := msg.(Unwrapper); ok { msg, err = w.Unwrap() if err != nil { @@ -247,12 +254,14 @@ func newPeer( p.metrics.MessageReceiveBytesTotal.With(append(labels, "message_type", p.mlc.ValueToMetricLabel(msg))...).Add(float64(len(msgBytes))) schema.WriteReceivedBytes(p.traceClient, string(p.ID()), chID, len(msgBytes)) if nr, ok := reactor.(EnvelopeReceiver); ok { + //p.Logger.Info("reactor.(EnvelopeReceiver)") nr.ReceiveEnvelope(Envelope{ ChannelID: chID, Src: p, Message: msg, }) } else { + //p.Logger.Info("not reactor.(EnvelopeReceiver)") reactor.Receive(chID, p, msgBytes) } } @@ -372,6 +381,7 @@ func (p *peer) SendEnvelope(e Envelope) bool { if w, ok := msg.(Wrapper); ok { msg = w.Wrap() } + p.Logger.Info("Sending envelope", "envelope", e) msgBytes, err := proto.Marshal(msg) if err != nil { p.Logger.Error("marshaling message to send", "error", err) @@ -400,32 +410,42 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { } stream, has := p.streams[chID] if !has { - newStream, err := p.conn.OpenStream() + newStream, err := p.conn.OpenStreamSync(context.Background()) if err != nil { p.Logger.Error("error opening quic stream", "err", err.Error()) return false } p.streams[chID] = newStream stream = newStream - _, err = stream.Write([]byte{chID}) + err = binary.Write(stream, binary.BigEndian, chID) if err != nil { p.Logger.Error("error sending channel ID", "err", err.Error()) return false } - p.Logger.Info("successfully shared channel ID", "id", chID) + //p.Logger.Info("successfully shared channel ID", "id", chID) } - p.Logger.Debug("Send", "channel", chID, "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) - n, err := stream.Write(msgBytes) - if err != nil { - p.Logger.Debug("Send failed", "channel", "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) - return false - } - labels := []string{ - "peer_id", string(p.ID()), - "chID", fmt.Sprintf("%#x", chID), - } - p.metrics.PeerSendBytesTotal.With(labels...).Add(float64(n)) + //p.Logger.Info("Send", "channel", chID, "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) + + go func() { + if err := binary.Write(stream, binary.BigEndian, uint32(len(msgBytes))); err != nil { + p.Logger.Error("Send len failed", "err", err, "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) + return + } + //p.Logger.Error("sending size of data", "size", len(msgBytes)) + err := binary.Write(stream, binary.BigEndian, msgBytes) + if err != nil { + p.Logger.Info("Send failed", "channel", "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) + return + } + //p.Logger.Info("sent data", "channel", chID, "len_data", len(msgBytes), "data", hex.EncodeToString(msgBytes)) + labels := []string{ + "peer_id", string(p.ID()), + "chID", fmt.Sprintf("%#x", chID), + } + p.metrics.PeerSendBytesTotal.With(labels...).Add(float64(len(msgBytes))) + }() + return true } @@ -538,27 +558,36 @@ func (p *peer) OnReceive() error { stream, err := p.conn.AcceptStream(context.Background()) if err != nil { p.Logger.Error("failed to accept stream", "err", err.Error()) + return errors.New("failed to accept stream") + } + var chID byte + err = binary.Read(stream, binary.BigEndian, &chID) + //_, err = stream.Read(chID) + if err != nil { + p.Logger.Error("failed to read channel ID", "err", err.Error()) return err } + p.streams[chID] = stream + p.Logger.Info("successfully got channel ID", "id", chID) + // start accepting data go func() { - chID := make([]byte, 1) - _, err := stream.Read(chID) - if err != nil { - p.Logger.Error("failed to read channel ID", "err", err.Error()) - return - } - p.streams[chID[0]] = stream - p.Logger.Info("successfully got channel ID", "id", chID[0]) - // start accepting data - go func() { - data := make([]byte, 1000000000) - n, err := stream.Read(data) + for { + var dataLen uint32 + err = binary.Read(stream, binary.BigEndian, &dataLen) + if err != nil { + p.Logger.Error("failed to read size from stream", "err", err.Error()) + return + } + //p.Logger.Error("received size", "size", dataLen) + data := make([]byte, dataLen) + _, err = io.ReadFull(stream, data) if err != nil { - p.Logger.Error("failed to read from stream", "err", err.Error()) + p.Logger.Error("failed to read data from stream", "err", err.Error()) return } - p.onReceive(chID[0], data[:n]) - }() + //p.Logger.Info("received data", "channel", chID[0], "len_data", dataLen, "data", hex.EncodeToString(data)) + p.onReceive(chID, data) + } }() } return errors.New("reached 500 opened streams with peer") diff --git a/p2p/switch.go b/p2p/switch.go index af0607e037..d2551ef910 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -32,17 +32,6 @@ const ( reconnectBackOffBaseSeconds = 3 ) -// MConnConfig returns an MConnConfig with fields updated -// from the P2PConfig. -func MConnConfig(cfg *config.P2PConfig) conn.MConnConfig { - mConfig := conn.DefaultMConnConfig() - mConfig.FlushThrottle = cfg.FlushThrottleTimeout - mConfig.SendRate = cfg.SendRate - mConfig.RecvRate = cfg.RecvRate - mConfig.MaxPacketMsgPayloadSize = cfg.MaxPacketMsgPayloadSize - return mConfig -} - //----------------------------------------------------------------------------- // An AddrBook represents an address book from the pex package, which is used diff --git a/p2p/transport.go b/p2p/transport.go index 0b15012693..2a325903e6 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -159,11 +159,6 @@ type MultiplexTransport struct { nodeKey NodeKey resolver IPResolver - // TODO(xla): This config is still needed as we parameterise peerConn and - // peer currently. All relevant configuration should be refactored into options - // with sane defaults. - mConfig conn.MConnConfig - // the tracer is passed to peers for collecting trace data tracer trace.Tracer } @@ -176,7 +171,6 @@ var _ transportLifecycle = (*MultiplexTransport)(nil) func NewMultiplexTransport( nodeInfo NodeInfo, nodeKey NodeKey, - mConfig conn.MConnConfig, tracer trace.Tracer, ) *MultiplexTransport { return &MultiplexTransport{ @@ -185,7 +179,6 @@ func NewMultiplexTransport( dialTimeout: defaultDialTimeout, filterTimeout: defaultFilterTimeout, handshakeTimeout: defaultHandshakeTimeout, - mConfig: mConfig, nodeInfo: nodeInfo, nodeKey: nodeKey, conns: NewConnSet(), @@ -525,7 +518,7 @@ func handshake( ) go func(errc chan<- error, c quic.Connection) { - stream, err := c.OpenStream() + stream, err := c.OpenStreamSync(context.Background()) if err != nil { errc <- err return diff --git a/p2p/types.go b/p2p/types.go index 7a741bd295..91975068af 100644 --- a/p2p/types.go +++ b/p2p/types.go @@ -7,7 +7,6 @@ import ( ) type ChannelDescriptor = conn.ChannelDescriptor -type ConnectionStatus = conn.ConnectionStatus // Envelope contains a message with sender routing info. type Envelope struct { diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index a15d528a64..82c4693913 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -158,10 +158,9 @@ type ResultDialPeers struct { // A peer type Peer struct { - NodeInfo p2p.DefaultNodeInfo `json:"node_info"` - IsOutbound bool `json:"is_outbound"` - ConnectionStatus p2p.ConnectionStatus `json:"connection_status"` - RemoteIP string `json:"remote_ip"` + NodeInfo p2p.DefaultNodeInfo `json:"node_info"` + IsOutbound bool `json:"is_outbound"` + RemoteIP string `json:"remote_ip"` } // Validators for a height. From 78e6326dfd476d2a1bef348fb97ae0e3951bfa68 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 19 Aug 2024 13:00:24 +0100 Subject: [PATCH 05/55] remove private key from node address --- blockchain/v0/pool.go | 2 +- blockchain/v1/peer.go | 2 +- blockchain/v2/scheduler.go | 2 +- node/node.go | 2 +- p2p/netaddress.go | 12 +++++------- p2p/peer.go | 15 ++++++--------- p2p/pex/pex_reactor.go | 2 -- p2p/transport.go | 16 +++++++++++++--- 8 files changed, 28 insertions(+), 25 deletions(-) diff --git a/blockchain/v0/pool.go b/blockchain/v0/pool.go index b3a09b1dc3..6463833c54 100644 --- a/blockchain/v0/pool.go +++ b/blockchain/v0/pool.go @@ -46,7 +46,7 @@ const ( maxDiffBetweenCurrentAndReceivedBlockHeight = 100 ) -var peerTimeout = 15 * time.Second // not const so we can override with tests +var peerTimeout = 3 * time.Minute // not const so we can override with tests /* Peers self report their heights when we join the block pool. diff --git a/blockchain/v1/peer.go b/blockchain/v1/peer.go index ad26585b30..1f75878c3b 100644 --- a/blockchain/v1/peer.go +++ b/blockchain/v1/peer.go @@ -194,7 +194,7 @@ func (peer *BpPeer) stopBlockResponseTimer() bool { func BpPeerDefaultParams() *BpPeerParams { return &BpPeerParams{ // Timeout for a peer to respond to a block request. - timeout: 15 * time.Second, + timeout: 15 * time.Minute, // Minimum recv rate to ensure we're receiving blocks from a peer fast // enough. If a peer is not sending data at at least that rate, we diff --git a/blockchain/v2/scheduler.go b/blockchain/v2/scheduler.go index 35166764fe..9ab05fafd3 100644 --- a/blockchain/v2/scheduler.go +++ b/blockchain/v2/scheduler.go @@ -209,7 +209,7 @@ func newScheduler(initHeight int64, startTime time.Time) *scheduler { pendingTime: make(map[int64]time.Time), receivedBlocks: make(map[int64]p2p.ID), targetPending: 10, // TODO - pass as param - peerTimeout: 15 * time.Second, // TODO - pass as param + peerTimeout: 15 * time.Minute, // TODO - pass as param minRecvRate: 0, // int64(7680), TODO - pass as param } diff --git a/node/node.go b/node/node.go index 6363b4e704..165c515a79 100644 --- a/node/node.go +++ b/node/node.go @@ -1058,7 +1058,7 @@ func (n *Node) OnStart() error { if err != nil { return err } - addr.PrivateKey = n.nodeKey.PrivKey + //addr.PrivateKey = n.nodeKey.PrivKey if err := n.transport.Listen(*addr); err != nil { return err } diff --git a/p2p/netaddress.go b/p2p/netaddress.go index 12da0bfab2..fb37630f95 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -11,7 +11,6 @@ import ( "errors" "fmt" "github.com/quic-go/quic-go" - "github.com/tendermint/tendermint/crypto" "net" "strconv" "strings" @@ -26,10 +25,9 @@ const EmptyNetAddress = "" // NetAddress defines information about a peer on the network // including its ID, IP address, and port. type NetAddress struct { - ID ID `json:"id"` - IP net.IP `json:"ip"` - Port uint16 `json:"port"` - PrivateKey crypto.PrivKey + ID ID `json:"id"` + IP net.IP `json:"ip"` + Port uint16 `json:"port"` } // IDAddressString returns id@hostPort. It strips the leading @@ -244,7 +242,7 @@ func (na *NetAddress) Dial() (quic.Connection, error) { MaxIdleTimeout: 10 * time.Minute, MaxIncomingStreams: 1000000000, MaxIncomingUniStreams: 1000000000, - KeepAlivePeriod: 100 * time.Millisecond, + KeepAlivePeriod: 15 * time.Second, EnableDatagrams: true, } conn, err := quic.DialAddr(context.Background(), na.DialString(), &tlsConfig, &quickConfig) @@ -263,7 +261,7 @@ func (na *NetAddress) DialTimeout(timeout time.Duration) (quic.Connection, error MaxIdleTimeout: 10 * time.Minute, MaxIncomingStreams: 1000000000, MaxIncomingUniStreams: 1000000000, - KeepAlivePeriod: 100 * time.Millisecond, + KeepAlivePeriod: 15 * time.Second, EnableDatagrams: true, } conn, err := quic.DialAddr(context.Background(), na.DialString(), &tlsConfig, &quickConfig) diff --git a/p2p/peer.go b/p2p/peer.go index 6393fcce58..14ce0d49f8 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -10,7 +10,6 @@ import ( "github.com/tendermint/tendermint/pkg/trace/schema" "io" "net" - "reflect" "time" "github.com/gogo/protobuf/proto" @@ -69,7 +68,6 @@ type EnvelopeSender interface { // // Deprecated: Will be removed in v0.37. func SendEnvelopeShim(p Peer, e Envelope, lg log.Logger) bool { - fmt.Println(e) if es, ok := p.(EnvelopeSender); ok { return es.SendEnvelope(e) } @@ -234,10 +232,10 @@ func newPeer( err := proto.Unmarshal(msgBytes, msg) if err != nil { p.Logger.Error("before panic", "msg", msg, "type", mt, "bytes", hex.EncodeToString(msgBytes)) - panic(fmt.Errorf("unmarshaling message: %s into type: %s", err, reflect.TypeOf(mt))) + return } - p.Logger.Error("type of message", "type", msg) + //p.Logger.Error("type of message", "type", msg) if w, ok := msg.(Unwrapper); ok { msg, err = w.Unwrap() if err != nil { @@ -422,7 +420,7 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { p.Logger.Error("error sending channel ID", "err", err.Error()) return false } - //p.Logger.Info("successfully shared channel ID", "id", chID) + //p.Logger.Error("successfully opened channel", "id", chID) } //p.Logger.Info("Send", "channel", chID, "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) @@ -438,7 +436,7 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { p.Logger.Info("Send failed", "channel", "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) return } - //p.Logger.Info("sent data", "channel", chID, "len_data", len(msgBytes), "data", hex.EncodeToString(msgBytes)) + p.Logger.Info("sent data", "channel", chID, "len_data", len(msgBytes)) labels := []string{ "peer_id", string(p.ID()), "chID", fmt.Sprintf("%#x", chID), @@ -567,8 +565,7 @@ func (p *peer) OnReceive() error { p.Logger.Error("failed to read channel ID", "err", err.Error()) return err } - p.streams[chID] = stream - p.Logger.Info("successfully got channel ID", "id", chID) + //p.Logger.Error("successfully opened channel", "id", chID) // start accepting data go func() { for { @@ -585,7 +582,7 @@ func (p *peer) OnReceive() error { p.Logger.Error("failed to read data from stream", "err", err.Error()) return } - //p.Logger.Info("received data", "channel", chID[0], "len_data", dataLen, "data", hex.EncodeToString(data)) + p.Logger.Info("received data", "channel", chID, "len_data", dataLen) p.onReceive(chID, data) } }() diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 8883728a6b..dad1c9e8fd 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -238,8 +238,6 @@ func (r *Reactor) logErrAddrBook(err error) { // Receive implements Reactor by handling incoming PEX messages. func (r *Reactor) ReceiveEnvelope(e p2p.Envelope) { - r.Logger.Debug("Received message", "src", e.Src, "chId", e.ChannelID, "msg", e.Message) - switch msg := e.Message.(type) { case *tmp2p.PexRequest: diff --git a/p2p/transport.go b/p2p/transport.go index 2a325903e6..7d251a8bb9 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -215,7 +215,7 @@ func (mt *MultiplexTransport) Dial( addr NetAddress, cfg peerConfig, ) (Peer, error) { - addr.PrivateKey = mt.nodeKey.PrivKey + //addr.PrivateKey = mt.nodeKey.PrivKey c, err := addr.DialTimeout(mt.dialTimeout) if err != nil { return nil, err @@ -261,7 +261,11 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { BasicConstraintsValid: true, } - edKey := ed25519.PrivateKey(addr.PrivateKey.Bytes()) + _, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return err + } + edKey := ed25519.PrivateKey(private) derBytes, err := x509.CreateCertificate(rand.Reader, &certTemplate, &certTemplate, edKey.Public(), edKey) if err != nil { return err @@ -273,7 +277,13 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { PrivateKey: edKey, }}, } - quickConfig := quic.Config{} + quickConfig := quic.Config{ + MaxIdleTimeout: 10 * time.Minute, + MaxIncomingStreams: 1000000000, + MaxIncomingUniStreams: 1000000000, + KeepAlivePeriod: 15 * time.Second, + EnableDatagrams: true, + } listener, err := quic.ListenAddr(addr.DialString(), &tlsConfig, &quickConfig) if err != nil { return err From 5ea01bc129ef4743ccfdb975a88ae32d9f4c380e Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 19 Aug 2024 15:20:04 +0100 Subject: [PATCH 06/55] rename to start receiving --- p2p/peer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index 14ce0d49f8..20a999b289 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -265,7 +265,7 @@ func newPeer( } go func() { - err := p.OnReceive() + err := p.StartReceiving() if err != nil { p.Logger.Error("error receiving stuff", "err", err.Error()) } @@ -551,7 +551,7 @@ func (p *peer) metricsReporter() { } } -func (p *peer) OnReceive() error { +func (p *peer) StartReceiving() error { for i := 0; i < 500; i++ { stream, err := p.conn.AcceptStream(context.Background()) if err != nil { From 07e9a095a0d1dd4fce62a0f0de256687112c6e8c Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 19 Aug 2024 16:16:33 +0100 Subject: [PATCH 07/55] synchronous send --- p2p/peer.go | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index 20a999b289..46c7e4cfb6 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -231,7 +231,7 @@ func newPeer( msg := proto.Clone(mt) err := proto.Unmarshal(msgBytes, msg) if err != nil { - p.Logger.Error("before panic", "msg", msg, "type", mt, "bytes", hex.EncodeToString(msgBytes)) + p.Logger.Error("before panic", "msg", msg, "channel", chID, "type", mt, "bytes", hex.EncodeToString(msgBytes)) return } @@ -425,24 +425,22 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { //p.Logger.Info("Send", "channel", chID, "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) - go func() { - if err := binary.Write(stream, binary.BigEndian, uint32(len(msgBytes))); err != nil { - p.Logger.Error("Send len failed", "err", err, "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) - return - } - //p.Logger.Error("sending size of data", "size", len(msgBytes)) - err := binary.Write(stream, binary.BigEndian, msgBytes) - if err != nil { - p.Logger.Info("Send failed", "channel", "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) - return - } - p.Logger.Info("sent data", "channel", chID, "len_data", len(msgBytes)) - labels := []string{ - "peer_id", string(p.ID()), - "chID", fmt.Sprintf("%#x", chID), - } - p.metrics.PeerSendBytesTotal.With(labels...).Add(float64(len(msgBytes))) - }() + if err := binary.Write(stream, binary.BigEndian, uint32(len(msgBytes))); err != nil { + p.Logger.Error("Send len failed", "err", err, "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) + return false + } + //p.Logger.Error("sending size of data", "size", len(msgBytes)) + err := binary.Write(stream, binary.BigEndian, msgBytes) + if err != nil { + p.Logger.Info("Send failed", "channel", "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) + return false + } + //p.Logger.Error("sent dataLen", "channel", chID, "len_data", len(msgBytes), "bytes", hex.EncodeToString(msgBytes)) + labels := []string{ + "peer_id", string(p.ID()), + "chID", fmt.Sprintf("%#x", chID), + } + p.metrics.PeerSendBytesTotal.With(labels...).Add(float64(len(msgBytes))) return true } @@ -582,7 +580,7 @@ func (p *peer) StartReceiving() error { p.Logger.Error("failed to read data from stream", "err", err.Error()) return } - p.Logger.Info("received data", "channel", chID, "len_data", dataLen) + //p.Logger.Info("received data", "channel", chID, "len_data", dataLen, "bytes", hex.EncodeToString(data)) p.onReceive(chID, data) } }() From 7e7098c06ef7002fae80c42a1fec8b0e2dc625b0 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 19 Aug 2024 17:41:20 +0100 Subject: [PATCH 08/55] cleanup --- p2p/peer.go | 14 +------------- p2p/transport.go | 5 ++--- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index 46c7e4cfb6..789d0f108f 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -220,7 +220,6 @@ func newPeer( } p.onReceive = func(chID byte, msgBytes []byte) { - //p.Logger.Info("handling data", "channel", chID, "data", hex.EncodeToString(msgBytes)) reactor := reactorsByCh[chID] if reactor == nil { // Note that its ok to panic here as it's caught in the conn._recover, @@ -235,7 +234,6 @@ func newPeer( return } - //p.Logger.Error("type of message", "type", msg) if w, ok := msg.(Unwrapper); ok { msg, err = w.Unwrap() if err != nil { @@ -252,14 +250,12 @@ func newPeer( p.metrics.MessageReceiveBytesTotal.With(append(labels, "message_type", p.mlc.ValueToMetricLabel(msg))...).Add(float64(len(msgBytes))) schema.WriteReceivedBytes(p.traceClient, string(p.ID()), chID, len(msgBytes)) if nr, ok := reactor.(EnvelopeReceiver); ok { - //p.Logger.Info("reactor.(EnvelopeReceiver)") nr.ReceiveEnvelope(Envelope{ ChannelID: chID, Src: p, Message: msg, }) } else { - //p.Logger.Info("not reactor.(EnvelopeReceiver)") reactor.Receive(chID, p, msgBytes) } } @@ -420,22 +416,17 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { p.Logger.Error("error sending channel ID", "err", err.Error()) return false } - //p.Logger.Error("successfully opened channel", "id", chID) } - //p.Logger.Info("Send", "channel", chID, "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) - if err := binary.Write(stream, binary.BigEndian, uint32(len(msgBytes))); err != nil { p.Logger.Error("Send len failed", "err", err, "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) return false } - //p.Logger.Error("sending size of data", "size", len(msgBytes)) err := binary.Write(stream, binary.BigEndian, msgBytes) if err != nil { p.Logger.Info("Send failed", "channel", "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) return false } - //p.Logger.Error("sent dataLen", "channel", chID, "len_data", len(msgBytes), "bytes", hex.EncodeToString(msgBytes)) labels := []string{ "peer_id", string(p.ID()), "chID", fmt.Sprintf("%#x", chID), @@ -550,7 +541,7 @@ func (p *peer) metricsReporter() { } func (p *peer) StartReceiving() error { - for i := 0; i < 500; i++ { + for { stream, err := p.conn.AcceptStream(context.Background()) if err != nil { p.Logger.Error("failed to accept stream", "err", err.Error()) @@ -573,17 +564,14 @@ func (p *peer) StartReceiving() error { p.Logger.Error("failed to read size from stream", "err", err.Error()) return } - //p.Logger.Error("received size", "size", dataLen) data := make([]byte, dataLen) _, err = io.ReadFull(stream, data) if err != nil { p.Logger.Error("failed to read data from stream", "err", err.Error()) return } - //p.Logger.Info("received data", "channel", chID, "len_data", dataLen, "bytes", hex.EncodeToString(data)) p.onReceive(chID, data) } }() } - return errors.New("reached 500 opened streams with peer") } diff --git a/p2p/transport.go b/p2p/transport.go index 7d251a8bb9..a6d240d2a1 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -265,8 +265,7 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { if err != nil { return err } - edKey := ed25519.PrivateKey(private) - derBytes, err := x509.CreateCertificate(rand.Reader, &certTemplate, &certTemplate, edKey.Public(), edKey) + derBytes, err := x509.CreateCertificate(rand.Reader, &certTemplate, &certTemplate, private.Public(), private) if err != nil { return err } @@ -274,7 +273,7 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { tlsConfig := tls.Config{ Certificates: []tls.Certificate{{ Certificate: [][]byte{derBytes}, - PrivateKey: edKey, + PrivateKey: private, }}, } quickConfig := quic.Config{ From fd0577f1894a83b594af87f57a24de5a5457d65c Mon Sep 17 00:00:00 2001 From: sweexordious Date: Tue, 20 Aug 2024 10:13:27 +0100 Subject: [PATCH 09/55] use some kind of status --- blockchain/v0/pool.go | 2 +- blockchain/v1/peer.go | 2 +- blockchain/v2/scheduler.go | 2 +- consensus/reactor.go | 2 +- libs/net/net.go | 17 ------------- node/node.go | 1 - p2p/netaddress.go | 28 ++++++++++++--------- p2p/peer.go | 50 +++++++++++++++++-------------------- p2p/pex/pex_reactor.go | 8 +++--- p2p/transport.go | 10 ++++++-- p2p/types.go | 7 ++++++ rpc/core/net.go | 8 +++--- rpc/core/types/responses.go | 7 +++--- 13 files changed, 71 insertions(+), 73 deletions(-) diff --git a/blockchain/v0/pool.go b/blockchain/v0/pool.go index 6463833c54..b3a09b1dc3 100644 --- a/blockchain/v0/pool.go +++ b/blockchain/v0/pool.go @@ -46,7 +46,7 @@ const ( maxDiffBetweenCurrentAndReceivedBlockHeight = 100 ) -var peerTimeout = 3 * time.Minute // not const so we can override with tests +var peerTimeout = 15 * time.Second // not const so we can override with tests /* Peers self report their heights when we join the block pool. diff --git a/blockchain/v1/peer.go b/blockchain/v1/peer.go index 1f75878c3b..ad26585b30 100644 --- a/blockchain/v1/peer.go +++ b/blockchain/v1/peer.go @@ -194,7 +194,7 @@ func (peer *BpPeer) stopBlockResponseTimer() bool { func BpPeerDefaultParams() *BpPeerParams { return &BpPeerParams{ // Timeout for a peer to respond to a block request. - timeout: 15 * time.Minute, + timeout: 15 * time.Second, // Minimum recv rate to ensure we're receiving blocks from a peer fast // enough. If a peer is not sending data at at least that rate, we diff --git a/blockchain/v2/scheduler.go b/blockchain/v2/scheduler.go index 9ab05fafd3..35166764fe 100644 --- a/blockchain/v2/scheduler.go +++ b/blockchain/v2/scheduler.go @@ -209,7 +209,7 @@ func newScheduler(initHeight int64, startTime time.Time) *scheduler { pendingTime: make(map[int64]time.Time), receivedBlocks: make(map[int64]p2p.ID), targetPending: 10, // TODO - pass as param - peerTimeout: 15 * time.Minute, // TODO - pass as param + peerTimeout: 15 * time.Second, // TODO - pass as param minRecvRate: 0, // int64(7680), TODO - pass as param } diff --git a/consensus/reactor.go b/consensus/reactor.go index 5649aa1129..34eea2d23c 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -350,7 +350,7 @@ func (conR *Reactor) ReceiveEnvelope(e p2p.Envelope) { if votes := ourVotes.ToProto(); votes != nil { eMsg.Votes = *votes } - if p2p.SendEnvelopeShim(e.Src, p2p.Envelope{ //nolint: staticcheck + if p2p.TrySendEnvelopeShim(e.Src, p2p.Envelope{ //nolint: staticcheck ChannelID: VoteSetBitsChannel, Message: eMsg, }, conR.Logger) { diff --git a/libs/net/net.go b/libs/net/net.go index a911dca129..bed7d73853 100644 --- a/libs/net/net.go +++ b/libs/net/net.go @@ -1,10 +1,6 @@ package net import ( - "context" - "crypto/tls" - "fmt" - "github.com/quic-go/quic-go" "net" "strings" ) @@ -17,19 +13,6 @@ func Connect(protoAddr string) (net.Conn, error) { return conn, err } -// Connect dials the given address and returns a net.Conn. The protoAddr argument should be prefixed with the protocol, -// eg. "tcp://127.0.0.1:8080" or "unix:///tmp/test.sock" -func QuicConnect(protoAddr string) (quic.Connection, error) { - proto, address := ProtocolAndAddress(protoAddr) - tlsConfig := tls.Config{ - InsecureSkipVerify: true, - } - // TODO better config - quickConfig := quic.Config{} - conn, err := quic.DialAddr(context.Background(), fmt.Sprintf("%s:%s", proto, address), &tlsConfig, &quickConfig) - return conn, err -} - // ProtocolAndAddress splits an address into the protocol and address components. // For instance, "tcp://127.0.0.1:8080" will be split into "tcp" and "127.0.0.1:8080". // If the address has no protocol prefix, the default is "tcp". diff --git a/node/node.go b/node/node.go index 165c515a79..c8bfceb79b 100644 --- a/node/node.go +++ b/node/node.go @@ -1058,7 +1058,6 @@ func (n *Node) OnStart() error { if err != nil { return err } - //addr.PrivateKey = n.nodeKey.PrivKey if err := n.transport.Listen(*addr); err != nil { return err } diff --git a/p2p/netaddress.go b/p2p/netaddress.go index fb37630f95..71539cab1c 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -9,6 +9,7 @@ import ( "crypto/tls" "encoding/hex" "errors" + "flag" "fmt" "github.com/quic-go/quic-go" "net" @@ -43,16 +44,16 @@ func IDAddressString(id ID, protocolHostPort string) string { // panic. Panics if ID is invalid. // TODO: socks proxies? func NewNetAddress(id ID, addr net.Addr) *NetAddress { - tcpAddr := addr.(*net.UDPAddr) - //if !ok { - // if flag.Lookup("test.v") == nil { // normal run - // panic(fmt.Sprintf("Only TCPAddrs are supported. Got: %v", addr)) - // } else { // in testing - // netAddr := NewNetAddressIPPort(net.IP("127.0.0.1"), 0) - // netAddr.ID = id - // return netAddr - // } - //} + tcpAddr, ok := addr.(*net.UDPAddr) + if !ok { + if flag.Lookup("test.v") == nil { // normal run + panic(fmt.Sprintf("Only UDPAddrs are supported. Got: %v", addr)) + } else { // in testing + netAddr := NewNetAddressIPPort(net.IP("127.0.0.1"), 0) + netAddr.ID = id + return netAddr + } + } if err := validateID(id); err != nil { panic(fmt.Sprintf("Invalid ID %v: %v (addr: %v)", id, err, addr)) @@ -234,7 +235,8 @@ func (na *NetAddress) DialString() string { } // Dial calls net.Dial on the address. -func (na *NetAddress) Dial() (quic.Connection, error) { +// TODO: add tls stuff here +func (na *NetAddress) Dial(ctx context.Context) (quic.Connection, error) { tlsConfig := tls.Config{ InsecureSkipVerify: true, } @@ -245,7 +247,7 @@ func (na *NetAddress) Dial() (quic.Connection, error) { KeepAlivePeriod: 15 * time.Second, EnableDatagrams: true, } - conn, err := quic.DialAddr(context.Background(), na.DialString(), &tlsConfig, &quickConfig) + conn, err := quic.DialAddr(ctx, na.DialString(), &tlsConfig, &quickConfig) if err != nil { return nil, err } @@ -253,6 +255,8 @@ func (na *NetAddress) Dial() (quic.Connection, error) { } // DialTimeout calls net.DialTimeout on the address. +// TODO(rach-id): timeout +// TODO(rach-id): tls config func (na *NetAddress) DialTimeout(timeout time.Duration) (quic.Connection, error) { tlsConfig := tls.Config{ InsecureSkipVerify: true, diff --git a/p2p/peer.go b/p2p/peer.go index 789d0f108f..ed6d663a9a 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -4,7 +4,6 @@ import ( "context" "encoding/binary" "encoding/hex" - "errors" "fmt" "github.com/quic-go/quic-go" "github.com/tendermint/tendermint/pkg/trace/schema" @@ -38,7 +37,7 @@ type Peer interface { CloseConn() error // close original connection NodeInfo() NodeInfo // peer's info - //Status() cmtconn.ConnectionStatus + Status() ConnectionStatus SocketAddr() *NetAddress // actual address of the socket // Deprecated: entities looking to act as peers should implement SendEnvelope instead. @@ -114,7 +113,7 @@ type peerConn struct { conn quic.Connection // source connection socketAddr *NetAddress - + created time.Time // time of creation // cached RemoteIP() ip net.IP } @@ -130,11 +129,12 @@ func newPeerConn( persistent: persistent, conn: conn, socketAddr: socketAddr, + created: time.Now(), } } // ID only exists for SecretConnection. -// NOTE: Will panic if conn is not *SecretConnection. +// TODO(rach-id): fix ID here func (pc peerConn) ID() ID { return ID(pc.conn.RemoteAddr().String()) } @@ -263,7 +263,7 @@ func newPeer( go func() { err := p.StartReceiving() if err != nil { - p.Logger.Error("error receiving stuff", "err", err.Error()) + p.Logger.Error("error starting peer receive routine", "err", err.Error()) } }() @@ -278,10 +278,10 @@ func newPeer( // String representation. func (p *peer) String() string { if p.outbound { - return fmt.Sprintf("Peer{%v %v out}", p.conn, p.ID()) + return fmt.Sprintf("Peer{%v %v out}", p.conn.RemoteAddr().String(), p.ID()) } - return fmt.Sprintf("Peer{%v %v in}", p.conn, p.ID()) + return fmt.Sprintf("Peer{%v %v in}", p.conn.RemoteAddr().String(), p.ID()) } //--------------------------------------------------- @@ -309,6 +309,7 @@ func (p *peer) FlushStop() { p.metricsTicker.Stop() p.BaseService.OnStop() for _, stream := range p.streams { + // TODO(rach-id): set valid error codes stream.CancelRead(quic.StreamErrorCode(1)) // stop everything and close the conn stream.CancelWrite(quic.StreamErrorCode(1)) // stop everything and close the conn } @@ -318,7 +319,8 @@ func (p *peer) FlushStop() { func (p *peer) OnStop() { p.metricsTicker.Stop() p.BaseService.OnStop() - if err := p.conn.CloseWithError(quic.ApplicationErrorCode(2), "stopping in peer.OnStop()"); err != nil { // stop everything and close the conn + // TODO(rach-id): set valid error code + if err := p.conn.CloseWithError(quic.ApplicationErrorCode(2), "stopping peer connection"); err != nil { // stop everything and close the conn p.Logger.Debug("Error while stopping peer", "err", err) } } @@ -354,10 +356,13 @@ func (p *peer) SocketAddr() *NetAddress { return p.socketAddr } -//// Status returns the peer's ConnectionStatus. -//func (p *peer) Status() cmtconn.ConnectionStatus { -// return cmtconn.ConnectionStatus{} -//} +// Status returns the peer's ConnectionStatus. +func (p *peer) Status() ConnectionStatus { + return ConnectionStatus{ + Duration: time.Since(p.created), + ConnectionState: p.conn.ConnectionState(), + } +} // SendEnvelope sends the message in the envelope on the channel specified by the // envelope. Returns false if the connection times out trying to place the message @@ -375,7 +380,6 @@ func (p *peer) SendEnvelope(e Envelope) bool { if w, ok := msg.(Wrapper); ok { msg = w.Wrap() } - p.Logger.Info("Sending envelope", "envelope", e) msgBytes, err := proto.Marshal(msg) if err != nil { p.Logger.Error("marshaling message to send", "error", err) @@ -445,7 +449,7 @@ func (p *peer) TrySend(chID byte, msgBytes []byte) bool { } else if !p.hasChannel(chID) { return false } - res := p.Send(chID, msgBytes) + res := p.TrySend(chID, msgBytes) if res { labels := []string{ "peer_id", string(p.ID()), @@ -488,7 +492,8 @@ func (p *peer) hasChannel(chID byte) bool { // CloseConn closes original connection. Used for cleaning up in cases where the peer had not been started at all. func (p *peer) CloseConn() error { - return p.conn.CloseWithError(quic.ApplicationErrorCode(1), "closed in peer.CloseConn()") + // TODO(rach-id): valid error code + return p.conn.CloseWithError(quic.ApplicationErrorCode(1), "closed peer connection") } func (p *peer) SetRemovalFailed() { @@ -505,7 +510,8 @@ func (p *peer) GetRemovalFailed() bool { // CloseConn closes the underlying connection func (pc *peerConn) CloseConn() { - pc.conn.CloseWithError(quic.ApplicationErrorCode(1), "closed in CloseConn") + // TODO(rach-id): valid error code + pc.conn.CloseWithError(quic.ApplicationErrorCode(1), "closed peer connection") } // RemoteAddr returns peer's remote network address. @@ -530,14 +536,6 @@ func PeerMetrics(metrics *Metrics) PeerOption { } func (p *peer) metricsReporter() { - for { - select { - case <-p.metricsTicker.C: - //fmt.Println("in metrics reporter <-p.metricsTicker.C") - case <-p.Quit(): - return - } - } } func (p *peer) StartReceiving() error { @@ -545,16 +543,14 @@ func (p *peer) StartReceiving() error { stream, err := p.conn.AcceptStream(context.Background()) if err != nil { p.Logger.Error("failed to accept stream", "err", err.Error()) - return errors.New("failed to accept stream") + return err } var chID byte err = binary.Read(stream, binary.BigEndian, &chID) - //_, err = stream.Read(chID) if err != nil { p.Logger.Error("failed to read channel ID", "err", err.Error()) return err } - //p.Logger.Error("successfully opened channel", "id", chID) // start accepting data go func() { for { diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index dad1c9e8fd..5fc24bfca2 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -238,6 +238,8 @@ func (r *Reactor) logErrAddrBook(err error) { // Receive implements Reactor by handling incoming PEX messages. func (r *Reactor) ReceiveEnvelope(e p2p.Envelope) { + r.Logger.Debug("Received message", "src", e.Src, "chId", e.ChannelID, "msg", e.Message) + switch msg := e.Message.(type) { case *tmp2p.PexRequest: @@ -761,9 +763,9 @@ func (r *Reactor) cleanupCrawlPeerInfos() { // attemptDisconnects checks if we've been with each peer long enough to disconnect func (r *Reactor) attemptDisconnects() { for _, peer := range r.Switch.Peers().List() { - //if peer.Status().Duration < r.config.SeedDisconnectWaitPeriod { - // continue - //} + if peer.Status().Duration < r.config.SeedDisconnectWaitPeriod { + continue + } if peer.IsPersistent() { continue } diff --git a/p2p/transport.go b/p2p/transport.go index a6d240d2a1..3a8e3a49fb 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -215,7 +215,6 @@ func (mt *MultiplexTransport) Dial( addr NetAddress, cfg peerConfig, ) (Peer, error) { - //addr.PrivateKey = mt.nodeKey.PrivKey c, err := addr.DialTimeout(mt.dialTimeout) if err != nil { return nil, err @@ -251,6 +250,7 @@ func (mt *MultiplexTransport) Close() error { // Listen implements transportLifecycle. func (mt *MultiplexTransport) Listen(addr NetAddress) error { + // TODO(rach-id): valid certificate serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) certTemplate := x509.Certificate{ @@ -276,6 +276,7 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { PrivateKey: private, }}, } + // TODO(rach-id): valid config quickConfig := quic.Config{ MaxIdleTimeout: 10 * time.Minute, MaxIncomingStreams: 1000000000, @@ -290,7 +291,7 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { mt.listener = listener mt.netAddr = addr - // TODO use a better context + // TODO(rach-id): use a better context go mt.acceptPeers(context.Background()) return nil @@ -344,6 +345,7 @@ func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { case mt.acceptc <- accept{err: err}: case <-mt.closec: // Give up if the transport was closed. + // TODO(rach-id): valid error code _ = c.CloseWithError(quic.ApplicationErrorCode(1), "some error 1") return } @@ -370,6 +372,7 @@ func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { // Make the upgraded peer available. case <-mt.closec: // Give up if the transport was closed. + // TODO(rach-id): valid error _ = c.CloseWithError(quic.ApplicationErrorCode(1), "some error 2") return } @@ -387,12 +390,14 @@ func (mt *MultiplexTransport) Cleanup(p Peer) { func (mt *MultiplexTransport) cleanup(c quic.Connection) error { mt.conns.Remove(c) + // TODO(rach-id): valid error return c.CloseWithError(quic.ApplicationErrorCode(1), "some error 3") } func (mt *MultiplexTransport) filterConn(c quic.Connection) (err error) { defer func() { if err != nil { + // TODO(rach-id): valid error _ = c.CloseWithError(quic.ApplicationErrorCode(1), "some error 4") } }() @@ -477,6 +482,7 @@ func (mt *MultiplexTransport) upgrade( } } + // TODO(rach-id): valid ID // Ensure connection key matches self reported key. //if connID != nodeInfo.ID() { // return nil, nil, ErrRejected{ diff --git a/p2p/types.go b/p2p/types.go index 91975068af..d4fb170fe1 100644 --- a/p2p/types.go +++ b/p2p/types.go @@ -2,12 +2,19 @@ package p2p import ( "github.com/gogo/protobuf/proto" + "github.com/quic-go/quic-go" "github.com/tendermint/tendermint/p2p/conn" tmp2p "github.com/tendermint/tendermint/proto/tendermint/p2p" + "time" ) type ChannelDescriptor = conn.ChannelDescriptor +type ConnectionStatus struct { + Duration time.Duration + ConnectionState quic.ConnectionState +} + // Envelope contains a message with sender routing info. type Envelope struct { Src Peer // sender (empty if outbound) diff --git a/rpc/core/net.go b/rpc/core/net.go index 1d07d594aa..57ae4f1b42 100644 --- a/rpc/core/net.go +++ b/rpc/core/net.go @@ -22,10 +22,10 @@ func NetInfo(ctx *rpctypes.Context) (*ctypes.ResultNetInfo, error) { return nil, fmt.Errorf("peer.NodeInfo() is not DefaultNodeInfo") } peers = append(peers, ctypes.Peer{ - NodeInfo: nodeInfo, - IsOutbound: peer.IsOutbound(), - //ConnectionStatus: peer.Status(), - RemoteIP: peer.RemoteIP().String(), + NodeInfo: nodeInfo, + IsOutbound: peer.IsOutbound(), + ConnectionStatus: peer.Status(), + RemoteIP: peer.RemoteIP().String(), }) } // TODO: Should we include PersistentPeers and Seeds in here? diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 82c4693913..a15d528a64 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -158,9 +158,10 @@ type ResultDialPeers struct { // A peer type Peer struct { - NodeInfo p2p.DefaultNodeInfo `json:"node_info"` - IsOutbound bool `json:"is_outbound"` - RemoteIP string `json:"remote_ip"` + NodeInfo p2p.DefaultNodeInfo `json:"node_info"` + IsOutbound bool `json:"is_outbound"` + ConnectionStatus p2p.ConnectionStatus `json:"connection_status"` + RemoteIP string `json:"remote_ip"` } // Validators for a height. From bde075bca55f348ea1bcf39bb823910011af327d Mon Sep 17 00:00:00 2001 From: sweexordious Date: Tue, 20 Aug 2024 11:47:37 +0100 Subject: [PATCH 10/55] fix send --- p2p/peer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/peer.go b/p2p/peer.go index ed6d663a9a..7e1f53aa12 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -449,7 +449,7 @@ func (p *peer) TrySend(chID byte, msgBytes []byte) bool { } else if !p.hasChannel(chID) { return false } - res := p.TrySend(chID, msgBytes) + res := p.Send(chID, msgBytes) if res { labels := []string{ "peer_id", string(p.ID()), From c3351aa597fd12d85f5b4bfe17e88bad286c7522 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Tue, 20 Aug 2024 11:53:23 +0100 Subject: [PATCH 11/55] min tls version and 0rtt comment --- p2p/netaddress.go | 2 ++ p2p/transport.go | 3 +++ 2 files changed, 5 insertions(+) diff --git a/p2p/netaddress.go b/p2p/netaddress.go index 71539cab1c..6e37bd5ff5 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -238,6 +238,7 @@ func (na *NetAddress) DialString() string { // TODO: add tls stuff here func (na *NetAddress) Dial(ctx context.Context) (quic.Connection, error) { tlsConfig := tls.Config{ + MinVersion: tls.VersionTLS13, InsecureSkipVerify: true, } quickConfig := quic.Config{ @@ -259,6 +260,7 @@ func (na *NetAddress) Dial(ctx context.Context) (quic.Connection, error) { // TODO(rach-id): tls config func (na *NetAddress) DialTimeout(timeout time.Duration) (quic.Connection, error) { tlsConfig := tls.Config{ + MinVersion: tls.VersionTLS13, InsecureSkipVerify: true, } quickConfig := quic.Config{ diff --git a/p2p/transport.go b/p2p/transport.go index 3a8e3a49fb..df0b5d4c24 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -271,6 +271,7 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { } tlsConfig := tls.Config{ + MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{{ Certificate: [][]byte{derBytes}, PrivateKey: private, @@ -278,6 +279,8 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { } // TODO(rach-id): valid config quickConfig := quic.Config{ + // TODO(rach-id): do we want to enable 0RTT? are the replay risks fine? + Allow0RTT: false, MaxIdleTimeout: 10 * time.Minute, MaxIncomingStreams: 1000000000, MaxIncomingUniStreams: 1000000000, From 6f401e9d107042eef6997ea679e9f53d5f0bafb4 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Tue, 20 Aug 2024 12:44:06 +0100 Subject: [PATCH 12/55] reduce number of streams per connection --- p2p/netaddress.go | 8 ++++---- p2p/transport.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/p2p/netaddress.go b/p2p/netaddress.go index 6e37bd5ff5..d7aa76bcc2 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -243,8 +243,8 @@ func (na *NetAddress) Dial(ctx context.Context) (quic.Connection, error) { } quickConfig := quic.Config{ MaxIdleTimeout: 10 * time.Minute, - MaxIncomingStreams: 1000000000, - MaxIncomingUniStreams: 1000000000, + MaxIncomingStreams: 10000, + MaxIncomingUniStreams: 10000, KeepAlivePeriod: 15 * time.Second, EnableDatagrams: true, } @@ -265,8 +265,8 @@ func (na *NetAddress) DialTimeout(timeout time.Duration) (quic.Connection, error } quickConfig := quic.Config{ MaxIdleTimeout: 10 * time.Minute, - MaxIncomingStreams: 1000000000, - MaxIncomingUniStreams: 1000000000, + MaxIncomingStreams: 10000, + MaxIncomingUniStreams: 10000, KeepAlivePeriod: 15 * time.Second, EnableDatagrams: true, } diff --git a/p2p/transport.go b/p2p/transport.go index df0b5d4c24..0bcd9f8186 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -282,8 +282,8 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { // TODO(rach-id): do we want to enable 0RTT? are the replay risks fine? Allow0RTT: false, MaxIdleTimeout: 10 * time.Minute, - MaxIncomingStreams: 1000000000, - MaxIncomingUniStreams: 1000000000, + MaxIncomingStreams: 10000, + MaxIncomingUniStreams: 10000, KeepAlivePeriod: 15 * time.Second, EnableDatagrams: true, } From b14191ececd31ab73d3ee460ddf29c61604bb900 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Tue, 20 Aug 2024 13:21:34 +0100 Subject: [PATCH 13/55] tls using rsa and freshly generated keys --- p2p/netaddress.go | 34 +++++++++++++++++++++++++++++++++- p2p/transport.go | 8 ++++---- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/p2p/netaddress.go b/p2p/netaddress.go index d7aa76bcc2..bdc2520933 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -6,12 +6,17 @@ package p2p import ( "context" + "crypto/rand" + "crypto/rsa" "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" "encoding/hex" "errors" "flag" "fmt" "github.com/quic-go/quic-go" + "math/big" "net" "strconv" "strings" @@ -235,7 +240,8 @@ func (na *NetAddress) DialString() string { } // Dial calls net.Dial on the address. -// TODO: add tls stuff here +// TODO: add TLS stuff. +// Note: this one is not used in the code, the DialTimeout is used instead. func (na *NetAddress) Dial(ctx context.Context) (quic.Connection, error) { tlsConfig := tls.Config{ MinVersion: tls.VersionTLS13, @@ -259,9 +265,35 @@ func (na *NetAddress) Dial(ctx context.Context) (quic.Connection, error) { // TODO(rach-id): timeout // TODO(rach-id): tls config func (na *NetAddress) DialTimeout(timeout time.Duration) (quic.Connection, error) { + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + certTemplate := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"Test"}, + }, + BasicConstraintsValid: true, + } + + // using rsa 4096 as ed25519 apparently is not supported and an error is returned. + // TODO(rach-id): investigate this + rsaPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, err + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &certTemplate, &certTemplate, rsaPrivateKey.Public(), rsaPrivateKey) + if err != nil { + return nil, err + } + tlsConfig := tls.Config{ MinVersion: tls.VersionTLS13, InsecureSkipVerify: true, + Certificates: []tls.Certificate{{ + Certificate: [][]byte{derBytes}, + PrivateKey: rsaPrivateKey, + }}, } quickConfig := quic.Config{ MaxIdleTimeout: 10 * time.Minute, diff --git a/p2p/transport.go b/p2p/transport.go index 0bcd9f8186..94a5086430 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -2,7 +2,7 @@ package p2p import ( "context" - "crypto/ed25519" + "crypto/rsa" "crypto/tls" "crypto/x509" "crypto/x509/pkix" @@ -261,11 +261,11 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { BasicConstraintsValid: true, } - _, private, err := ed25519.GenerateKey(rand.Reader) + rsaPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096) if err != nil { return err } - derBytes, err := x509.CreateCertificate(rand.Reader, &certTemplate, &certTemplate, private.Public(), private) + derBytes, err := x509.CreateCertificate(rand.Reader, &certTemplate, &certTemplate, rsaPrivateKey.Public(), rsaPrivateKey) if err != nil { return err } @@ -274,7 +274,7 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{{ Certificate: [][]byte{derBytes}, - PrivateKey: private, + PrivateKey: rsaPrivateKey, }}, } // TODO(rach-id): valid config From ee6ff452997df83d0261aa556dc51f5dd93ef7f4 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Tue, 20 Aug 2024 13:39:00 +0100 Subject: [PATCH 14/55] add peer reconnection todo --- p2p/transport.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/p2p/transport.go b/p2p/transport.go index 94a5086430..c9f0c6f454 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -221,9 +221,10 @@ func (mt *MultiplexTransport) Dial( } // TODO(xla): Evaluate if we should apply filters if we explicitly dial. - if err := mt.filterConn(c); err != nil { - return nil, err - } + //TODO(rach-id): remove peer when disconnected. + //if err := mt.filterConn(c); err != nil { + // return nil, err + //} secretConn, nodeInfo, err := mt.upgrade(c, &addr) if err != nil { From d93bef9ca8463f6a8981cda96fec02129f4380de Mon Sep 17 00:00:00 2001 From: sweexordious Date: Tue, 20 Aug 2024 17:56:27 +0100 Subject: [PATCH 15/55] close connection after 5 sec --- p2p/netaddress.go | 8 ++++---- p2p/peer.go | 4 +++- p2p/transport.go | 11 ++++++----- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/p2p/netaddress.go b/p2p/netaddress.go index bdc2520933..c37f8d7d80 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -248,10 +248,10 @@ func (na *NetAddress) Dial(ctx context.Context) (quic.Connection, error) { InsecureSkipVerify: true, } quickConfig := quic.Config{ - MaxIdleTimeout: 10 * time.Minute, + MaxIdleTimeout: 5 * time.Second, MaxIncomingStreams: 10000, MaxIncomingUniStreams: 10000, - KeepAlivePeriod: 15 * time.Second, + KeepAlivePeriod: time.Second, EnableDatagrams: true, } conn, err := quic.DialAddr(ctx, na.DialString(), &tlsConfig, &quickConfig) @@ -296,10 +296,10 @@ func (na *NetAddress) DialTimeout(timeout time.Duration) (quic.Connection, error }}, } quickConfig := quic.Config{ - MaxIdleTimeout: 10 * time.Minute, + MaxIdleTimeout: 5 * time.Second, MaxIncomingStreams: 10000, MaxIncomingUniStreams: 10000, - KeepAlivePeriod: 15 * time.Second, + KeepAlivePeriod: time.Second, EnableDatagrams: true, } conn, err := quic.DialAddr(context.Background(), na.DialString(), &tlsConfig, &quickConfig) diff --git a/p2p/peer.go b/p2p/peer.go index 7e1f53aa12..3c2b089331 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -205,6 +205,7 @@ func newPeer( reactorsByCh map[byte]Reactor, msgTypeByChID map[byte]proto.Message, mlc *metricsLabelCache, + onPeerError func(Peer, interface{}), options ...PeerOption, ) *peer { p := &peer{ @@ -263,7 +264,8 @@ func newPeer( go func() { err := p.StartReceiving() if err != nil { - p.Logger.Error("error starting peer receive routine", "err", err.Error()) + p.Logger.Error("error when receiving data from peer", "err", err.Error()) + onPeerError(p, err) } }() diff --git a/p2p/transport.go b/p2p/transport.go index c9f0c6f454..75eedfa73a 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -222,9 +222,9 @@ func (mt *MultiplexTransport) Dial( // TODO(xla): Evaluate if we should apply filters if we explicitly dial. //TODO(rach-id): remove peer when disconnected. - //if err := mt.filterConn(c); err != nil { - // return nil, err - //} + if err := mt.filterConn(c); err != nil { + return nil, err + } secretConn, nodeInfo, err := mt.upgrade(c, &addr) if err != nil { @@ -282,10 +282,10 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { quickConfig := quic.Config{ // TODO(rach-id): do we want to enable 0RTT? are the replay risks fine? Allow0RTT: false, - MaxIdleTimeout: 10 * time.Minute, + MaxIdleTimeout: 5 * time.Second, MaxIncomingStreams: 10000, MaxIncomingUniStreams: 10000, - KeepAlivePeriod: 15 * time.Second, + KeepAlivePeriod: 100 * time.Millisecond, EnableDatagrams: true, } listener, err := quic.ListenAddr(addr.DialString(), &tlsConfig, &quickConfig) @@ -603,6 +603,7 @@ func (mt *MultiplexTransport) wrapPeer( cfg.reactorsByCh, cfg.msgTypeByChID, cfg.mlc, + cfg.onPeerError, PeerMetrics(cfg.metrics), WithPeerTracer(mt.tracer), ) From fcc4a137593361ae595ed4e6691413da89dd6ce4 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Tue, 27 Aug 2024 14:55:02 +0400 Subject: [PATCH 16/55] add TLS support --- p2p/certificate.go | 209 ++++++++++++++++++++++++++++++++++++++++++ p2p/netaddress.go | 43 +-------- p2p/peer_test.go | 6 +- p2p/test_util.go | 2 +- p2p/transport.go | 148 ++++++++++++++---------------- p2p/transport_test.go | 4 +- 6 files changed, 288 insertions(+), 124 deletions(-) create mode 100644 p2p/certificate.go diff --git a/p2p/certificate.go b/p2p/certificate.go new file mode 100644 index 0000000000..3c99110306 --- /dev/null +++ b/p2p/certificate.go @@ -0,0 +1,209 @@ +package p2p + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "errors" + "fmt" + "github.com/gogo/protobuf/proto" + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/encoding" + crypto2 "github.com/tendermint/tendermint/proto/tendermint/crypto" + "math/big" + "time" +) + +// TODO(rach-id): mention this code is adapted from libp2p p2p/security/tls/crypto.go + +const certValidityPeriod = 24 * time.Hour +const certificatePrefix = "tendermint-tls:" + +// TODO(rach-id): update the OID prefix to reflect Celestia/Tendermint +var extensionPrefix = []int{1, 3, 6, 1, 4, 1, 53594} + +// getPrefixedExtensionID returns an Object Identifier +// that can be used in x509 Certificates. +func getPrefixedExtensionID(suffix []int) []int { + return append(extensionPrefix, suffix...) +} + +var extensionID = getPrefixedExtensionID([]int{1, 1}) +var extensionCritical bool // so we can mark the extension critical in tests + +// extensionIDEqual compares two extension IDs. +func extensionIDEqual(a, b []int) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +type signedKey struct { + PubKey []byte + Signature []byte +} + +// NewTLSConfig creates a new TLS configuration +func NewTLSConfig(privKey crypto.PrivKey) (*tls.Config, error) { + template, err := certTemplate() + if err != nil { + return nil, err + } + cert, err := keyToCertificate(privKey, template) + if err != nil { + return nil, err + } + return &tls.Config{ + MinVersion: tls.VersionTLS13, + InsecureSkipVerify: true, // This is not insecure here. We will verify the cert chain ourselves. + ClientAuth: tls.RequireAnyClientCert, + Certificates: []tls.Certificate{*cert}, + VerifyPeerCertificate: func(_ [][]byte, _ [][]*x509.Certificate) error { + panic("tls config not specialized for peer") + }, + }, nil +} + +// VerifyCertificate verifies the certificate chain and extract the remote's public key. +func VerifyCertificate(cert *x509.Certificate) (crypto.PubKey, error) { + pool := x509.NewCertPool() + pool.AddCert(cert) + var found bool + var keyExt pkix.Extension + // find the tendermint key extension, skipping all unknown extensions + for _, ext := range cert.Extensions { + if extensionIDEqual(ext.Id, extensionID) { + keyExt = ext + found = true + for i, oident := range cert.UnhandledCriticalExtensions { + if oident.Equal(ext.Id) { + // delete the extension from UnhandledCriticalExtensions + cert.UnhandledCriticalExtensions = append(cert.UnhandledCriticalExtensions[:i], cert.UnhandledCriticalExtensions[i+1:]...) + break + } + } + break + } + } + if !found { + return nil, errors.New("expected certificate to contain the key extension") + } + if _, err := cert.Verify(x509.VerifyOptions{Roots: pool}); err != nil { + // If we return an x509 error here, it will be sent on the wire. + // Wrap the error to avoid that. + return nil, fmt.Errorf("certificate verification failed: %s", err) + } + + var sk signedKey + if _, err := asn1.Unmarshal(keyExt.Value, &sk); err != nil { + return nil, fmt.Errorf("unmarshalling signed certificate failed: %s", err) + } + protoPubKey := crypto2.PublicKey{} + err := proto.Unmarshal(sk.PubKey, &protoPubKey) + if err != nil { + return nil, fmt.Errorf("unmarshalling public key failed: %s", err) + } + certKeyPub, err := x509.MarshalPKIXPublicKey(cert.PublicKey) + if err != nil { + return nil, err + } + pubKey, err := encoding.PubKeyFromProto(protoPubKey) + valid := pubKey.VerifySignature(append([]byte(certificatePrefix), certKeyPub...), sk.Signature) + if err != nil { + return nil, fmt.Errorf("signature verification failed: %s", err) + } + if !valid { + return nil, errors.New("signature invalid") + } + return pubKey, nil +} + +// GenerateSignedExtension uses the provided private key to sign the public key, and returns the +// signature within a pkix.Extension. +// This extension is included in a certificate to cryptographically tie it to the libp2p private key. +func GenerateSignedExtension(nodePrivateKey crypto.PrivKey, certificatePublicKey *ecdsa.PublicKey) (pkix.Extension, error) { + protoPubKey, err := encoding.PubKeyToProto(nodePrivateKey.PubKey()) + if err != nil { + return pkix.Extension{}, err + } + keyBytes, err := proto.Marshal(&protoPubKey) + if err != nil { + return pkix.Extension{}, err + } + certKeyPub, err := x509.MarshalPKIXPublicKey(certificatePublicKey) + if err != nil { + return pkix.Extension{}, err + } + signature, err := nodePrivateKey.Sign(append([]byte(certificatePrefix), certKeyPub...)) + if err != nil { + return pkix.Extension{}, err + } + value, err := asn1.Marshal(signedKey{ + PubKey: keyBytes, + Signature: signature, + }) + if err != nil { + return pkix.Extension{}, err + } + + return pkix.Extension{Id: extensionID, Critical: extensionCritical, Value: value}, nil +} + +// keyToCertificate generates a new ECDSA private key and corresponding x509 certificate. +// The certificate includes an extension that cryptographically ties it to the provided libp2p +// private key to authenticate TLS connections. +func keyToCertificate(sk crypto.PrivKey, certTmpl *x509.Certificate) (*tls.Certificate, error) { + certKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, err + } + + // after calling CreateCertificate, these will end up in Certificate.Extensions + extension, err := GenerateSignedExtension(sk, &certKey.PublicKey) + if err != nil { + return nil, err + } + certTmpl.ExtraExtensions = append(certTmpl.ExtraExtensions, extension) + + certDER, err := x509.CreateCertificate(rand.Reader, certTmpl, certTmpl, certKey.Public(), certKey) + if err != nil { + return nil, err + } + return &tls.Certificate{ + Certificate: [][]byte{certDER}, + PrivateKey: certKey, + }, nil +} + +// certTemplate returns the template for generating an Identity's TLS certificates. +func certTemplate() (*x509.Certificate, error) { + bigNum := big.NewInt(1 << 62) + sn, err := rand.Int(rand.Reader, bigNum) + if err != nil { + return nil, err + } + + subjectSN, err := rand.Int(rand.Reader, bigNum) + if err != nil { + return nil, err + } + + return &x509.Certificate{ + SerialNumber: sn, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(certValidityPeriod), + // According to RFC 3280, the issuer field must be set, + // see https://datatracker.ietf.org/doc/html/rfc3280#section-4.1.2.4. + Subject: pkix.Name{SerialNumber: subjectSN.String()}, + }, nil +} diff --git a/p2p/netaddress.go b/p2p/netaddress.go index c37f8d7d80..872c1a9148 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -6,17 +6,12 @@ package p2p import ( "context" - "crypto/rand" - "crypto/rsa" "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" "encoding/hex" "errors" "flag" "fmt" "github.com/quic-go/quic-go" - "math/big" "net" "strconv" "strings" @@ -262,47 +257,15 @@ func (na *NetAddress) Dial(ctx context.Context) (quic.Connection, error) { } // DialTimeout calls net.DialTimeout on the address. -// TODO(rach-id): timeout -// TODO(rach-id): tls config -func (na *NetAddress) DialTimeout(timeout time.Duration) (quic.Connection, error) { - serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) - serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) - certTemplate := x509.Certificate{ - SerialNumber: serialNumber, - Subject: pkix.Name{ - Organization: []string{"Test"}, - }, - BasicConstraintsValid: true, - } - - // using rsa 4096 as ed25519 apparently is not supported and an error is returned. - // TODO(rach-id): investigate this - rsaPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - return nil, err - } - - derBytes, err := x509.CreateCertificate(rand.Reader, &certTemplate, &certTemplate, rsaPrivateKey.Public(), rsaPrivateKey) - if err != nil { - return nil, err - } - - tlsConfig := tls.Config{ - MinVersion: tls.VersionTLS13, - InsecureSkipVerify: true, - Certificates: []tls.Certificate{{ - Certificate: [][]byte{derBytes}, - PrivateKey: rsaPrivateKey, - }}, - } +func (na *NetAddress) DialTimeout(tlsConf *tls.Config) (quic.Connection, error) { quickConfig := quic.Config{ MaxIdleTimeout: 5 * time.Second, MaxIncomingStreams: 10000, MaxIncomingUniStreams: 10000, - KeepAlivePeriod: time.Second, + KeepAlivePeriod: 100 * time.Millisecond, EnableDatagrams: true, } - conn, err := quic.DialAddr(context.Background(), na.DialString(), &tlsConfig, &quickConfig) + conn, err := quic.DialAddr(context.Background(), na.DialString(), tlsConf, &quickConfig) if err != nil { return nil, err } diff --git a/p2p/peer_test.go b/p2p/peer_test.go index 16fc653630..526992a729 100644 --- a/p2p/peer_test.go +++ b/p2p/peer_test.go @@ -94,7 +94,7 @@ func createOutboundPeerAndPerformHandshake( } timeout := 1 * time.Second ourNodeInfo := testNodeInfo(addr.ID, "host_peer") - peerNodeInfo, err := handshake(pc.conn, timeout, ourNodeInfo) + peerNodeInfo, err := exchangeNodeInfo(pc.conn, timeout, ourNodeInfo) if err != nil { return nil, err } @@ -195,7 +195,7 @@ func (rp *remotePeer) Dial(addr *NetAddress) (net.Conn, error) { if err != nil { return nil, err } - _, err = handshake(pc.conn, time.Second, rp.nodeInfo()) + _, err = exchangeNodeInfo(pc.conn, time.Second, rp.nodeInfo()) if err != nil { return nil, err } @@ -220,7 +220,7 @@ func (rp *remotePeer) accept() { golog.Fatalf("Failed to create a peer: %+v", err) } - _, err = handshake(pc.conn, time.Second, rp.nodeInfo()) + _, err = exchangeNodeInfo(pc.conn, time.Second, rp.nodeInfo()) if err != nil { golog.Fatalf("Failed to perform handshake: %+v", err) } diff --git a/p2p/test_util.go b/p2p/test_util.go index 147aa89ef8..ae247ea030 100644 --- a/p2p/test_util.go +++ b/p2p/test_util.go @@ -144,7 +144,7 @@ func (sw *Switch) addPeerWithConnection(conn net.Conn) error { return err } - ni, err := handshake(conn, time.Second, sw.nodeInfo) + ni, err := exchangeNodeInfo(conn, time.Second, sw.nodeInfo) if err != nil { if err := conn.Close(); err != nil { sw.Logger.Error("Error closing connection", "err", err) diff --git a/p2p/transport.go b/p2p/transport.go index 75eedfa73a..a941444511 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -2,19 +2,17 @@ package p2p import ( "context" - "crypto/rsa" - "crypto/tls" "crypto/x509" - "crypto/x509/pkix" + "errors" "fmt" "github.com/quic-go/quic-go" "github.com/tendermint/tendermint/libs/protoio" tmp2p "github.com/tendermint/tendermint/proto/tendermint/p2p" - "math/big" "net" + "os" + "runtime/debug" "time" - "crypto/rand" "github.com/gogo/protobuf/proto" "github.com/tendermint/tendermint/p2p/conn" "github.com/tendermint/tendermint/pkg/trace" @@ -215,7 +213,38 @@ func (mt *MultiplexTransport) Dial( addr NetAddress, cfg peerConfig, ) (Peer, error) { - c, err := addr.DialTimeout(mt.dialTimeout) + tlsConfig, err := NewTLSConfig(mt.nodeKey.PrivKey) + if err != nil { + return nil, err + } + tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) (err error) { + defer func() { + if rerr := recover(); rerr != nil { + fmt.Fprintf(os.Stderr, "panic when processing peer certificate in TLS exchangeNodeInfo: %s\n%s\n", rerr, debug.Stack()) + err = fmt.Errorf("panic when processing peer certificate in TLS exchangeNodeInfo: %s", rerr) + } + }() + + if len(rawCerts) != 1 { + return errors.New("expected one certificates in the chain") + } + cert, err := x509.ParseCertificate(rawCerts[0]) + if err != nil { + return err + } + + pubKey, err := VerifyCertificate(cert) + if err != nil { + return err + } + remoteID := PubKeyToID(pubKey) + if remoteID != addr.ID { + return fmt.Errorf("mismatch peer ID") + } + return nil + } + + c, err := addr.DialTimeout(tlsConfig) if err != nil { return nil, err } @@ -226,7 +255,7 @@ func (mt *MultiplexTransport) Dial( return nil, err } - secretConn, nodeInfo, err := mt.upgrade(c, &addr) + secretConn, nodeInfo, err := mt.getNodeInfo(c) if err != nil { return nil, err } @@ -251,34 +280,32 @@ func (mt *MultiplexTransport) Close() error { // Listen implements transportLifecycle. func (mt *MultiplexTransport) Listen(addr NetAddress) error { - // TODO(rach-id): valid certificate - serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) - serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) - certTemplate := x509.Certificate{ - SerialNumber: serialNumber, - Subject: pkix.Name{ - Organization: []string{"Test"}, - }, - BasicConstraintsValid: true, - } - - rsaPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - return err - } - derBytes, err := x509.CreateCertificate(rand.Reader, &certTemplate, &certTemplate, rsaPrivateKey.Public(), rsaPrivateKey) + tlsConfig, err := NewTLSConfig(mt.nodeKey.PrivKey) if err != nil { return err } + tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) (err error) { + defer func() { + if rerr := recover(); rerr != nil { + fmt.Fprintf(os.Stderr, "panic when processing peer certificate in TLS exchangeNodeInfo: %s\n%s\n", rerr, debug.Stack()) + err = fmt.Errorf("panic when processing peer certificate in TLS exchangeNodeInfo: %s", rerr) + } + }() - tlsConfig := tls.Config{ - MinVersion: tls.VersionTLS13, - Certificates: []tls.Certificate{{ - Certificate: [][]byte{derBytes}, - PrivateKey: rsaPrivateKey, - }}, + if len(rawCerts) != 1 { + return errors.New("expected one certificates in the chain") + } + cert, err := x509.ParseCertificate(rawCerts[0]) + if err != nil { + return err + } + + _, err = VerifyCertificate(cert) + if err != nil { + return err + } + return nil } - // TODO(rach-id): valid config quickConfig := quic.Config{ // TODO(rach-id): do we want to enable 0RTT? are the replay risks fine? Allow0RTT: false, @@ -288,7 +315,7 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { KeepAlivePeriod: 100 * time.Millisecond, EnableDatagrams: true, } - listener, err := quic.ListenAddr(addr.DialString(), &tlsConfig, &quickConfig) + listener, err := quic.ListenAddr(addr.DialString(), tlsConfig, &quickConfig) if err != nil { return err } @@ -363,7 +390,7 @@ func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { err := mt.filterConn(c) if err == nil { - _, nodeInfo, err = mt.upgrade(c, nil) + _, nodeInfo, err = mt.getNodeInfo(c) if err == nil { addr := c.RemoteAddr() id := PubKeyToID(mt.nodeKey.PubKey()) @@ -442,43 +469,23 @@ func (mt *MultiplexTransport) filterConn(c quic.Connection) (err error) { return nil } -func (mt *MultiplexTransport) upgrade( - c quic.Connection, - dialedAddr *NetAddress, -) (conn quic.Connection, nodeInfo NodeInfo, err error) { +func (mt *MultiplexTransport) getNodeInfo(c quic.Connection) (conn quic.Connection, remoteNodeInfo NodeInfo, err error) { defer func() { if err != nil { _ = mt.cleanup(c) } }() - // For outgoing conns, ensure connection key matches dialed key. - //connID := PubKeyToID(mt.nodeKey.PubKey()) - //if dialedAddr != nil { - // if dialedID := dialedAddr.ID; connID != dialedID { - // return nil, nil, ErrRejected{ - // conn: c, - // id: connID, - // err: fmt.Errorf( - // "conn.ID (%v) dialed ID (%v) mismatch", - // connID, - // dialedID, - // ), - // isAuthFailure: true, - // } - // } - //} - - nodeInfo, err = handshake(c, mt.handshakeTimeout, mt.nodeInfo) + remoteNodeInfo, err = exchangeNodeInfo(c, mt.handshakeTimeout, mt.nodeInfo) if err != nil { return nil, nil, ErrRejected{ conn: c, - err: fmt.Errorf("handshake failed: %v", err), + err: fmt.Errorf("exchangeNodeInfo failed: %v", err), isAuthFailure: true, } } - if err := nodeInfo.Validate(); err != nil { + if err := remoteNodeInfo.Validate(); err != nil { return nil, nil, ErrRejected{ conn: c, err: err, @@ -486,44 +493,29 @@ func (mt *MultiplexTransport) upgrade( } } - // TODO(rach-id): valid ID - // Ensure connection key matches self reported key. - //if connID != nodeInfo.ID() { - // return nil, nil, ErrRejected{ - // conn: c, - // id: connID, - // err: fmt.Errorf( - // "conn.ID (%v) NodeInfo.ID (%v) mismatch", - // connID, - // nodeInfo.ID(), - // ), - // isAuthFailure: true, - // } - //} - // Reject self. - if mt.nodeInfo.ID() == nodeInfo.ID() { + if mt.nodeInfo.ID() == remoteNodeInfo.ID() { return nil, nil, ErrRejected{ - addr: *NewNetAddress(nodeInfo.ID(), c.RemoteAddr()), + addr: *NewNetAddress(remoteNodeInfo.ID(), c.RemoteAddr()), conn: c, - id: nodeInfo.ID(), + id: remoteNodeInfo.ID(), isSelf: true, } } - if err := mt.nodeInfo.CompatibleWith(nodeInfo); err != nil { + if err := mt.nodeInfo.CompatibleWith(remoteNodeInfo); err != nil { return nil, nil, ErrRejected{ conn: c, err: err, - id: nodeInfo.ID(), + id: remoteNodeInfo.ID(), isIncompatible: true, } } - return c, nodeInfo, nil + return c, remoteNodeInfo, nil } -func handshake( +func exchangeNodeInfo( c quic.Connection, timeout time.Duration, nodeInfo NodeInfo, diff --git a/p2p/transport_test.go b/p2p/transport_test.go index 151ac7edf7..430da19d93 100644 --- a/p2p/transport_test.go +++ b/p2p/transport_test.go @@ -306,7 +306,7 @@ func TestTransportMultiplexAcceptNonBlocking(t *testing.T) { return } - _, err = handshake(sc, 200*time.Millisecond, + _, err = exchangeNodeInfo(sc, 200*time.Millisecond, testNodeInfo( PubKeyToID(ed25519.GenPrivKey().PubKey()), "slow_peer", @@ -612,7 +612,7 @@ func TestTransportHandshake(t *testing.T) { t.Fatal(err) } - ni, err := handshake(c, 20*time.Millisecond, emptyNodeInfo()) + ni, err := exchangeNodeInfo(c, 20*time.Millisecond, emptyNodeInfo()) if err != nil { t.Fatal(err) } From e53f0f9138ad2ec907c5fd85c9de0f0383b7ca60 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Tue, 27 Aug 2024 15:12:39 +0400 Subject: [PATCH 17/55] todos --- p2p/peer.go | 3 ++- p2p/transport.go | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index 3c2b089331..be9e73bd5a 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -361,7 +361,8 @@ func (p *peer) SocketAddr() *NetAddress { // Status returns the peer's ConnectionStatus. func (p *peer) Status() ConnectionStatus { return ConnectionStatus{ - Duration: time.Since(p.created), + Duration: time.Since(p.created), + // TODO(rach-id): register ecdsa.PublicKey protobuf definition ConnectionState: p.conn.ConnectionState(), } } diff --git a/p2p/transport.go b/p2p/transport.go index a941444511..37628c0936 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -250,7 +250,6 @@ func (mt *MultiplexTransport) Dial( } // TODO(xla): Evaluate if we should apply filters if we explicitly dial. - //TODO(rach-id): remove peer when disconnected. if err := mt.filterConn(c); err != nil { return nil, err } From c8f39ed429817df0bf815a8f2d469bbaf5b7ce17 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Wed, 28 Aug 2024 13:22:41 +0400 Subject: [PATCH 18/55] starting to fix e2e --- p2p/conn_set.go | 6 +++ p2p/netaddress.go | 8 ++-- p2p/peer.go | 8 ++-- p2p/transport.go | 26 ++++++----- test/maverick/node/node.go | 90 +++++++++++++++++++------------------- 5 files changed, 74 insertions(+), 64 deletions(-) diff --git a/p2p/conn_set.go b/p2p/conn_set.go index 1fa5547540..4f04568080 100644 --- a/p2p/conn_set.go +++ b/p2p/conn_set.go @@ -1,6 +1,7 @@ package p2p import ( + "fmt" "github.com/quic-go/quic-go" "net" @@ -38,6 +39,11 @@ func (cs *connSet) Has(c quic.Connection) bool { cs.RLock() defer cs.RUnlock() + fmt.Println("has") + for key, val := range cs.conns { + fmt.Printf("%s:%v\n", key, val.ips) + } + fmt.Printf("looking for %s\n", c.RemoteAddr().String()) _, ok := cs.conns[c.RemoteAddr().String()] return ok diff --git a/p2p/netaddress.go b/p2p/netaddress.go index 872c1a9148..98e5df00a5 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -257,15 +257,17 @@ func (na *NetAddress) Dial(ctx context.Context) (quic.Connection, error) { } // DialTimeout calls net.DialTimeout on the address. -func (na *NetAddress) DialTimeout(tlsConf *tls.Config) (quic.Connection, error) { +func (na *NetAddress) DialTimeout(timeout time.Duration, tlsConf *tls.Config) (quic.Connection, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() quickConfig := quic.Config{ - MaxIdleTimeout: 5 * time.Second, + MaxIdleTimeout: 200 * time.Millisecond, MaxIncomingStreams: 10000, MaxIncomingUniStreams: 10000, KeepAlivePeriod: 100 * time.Millisecond, EnableDatagrams: true, } - conn, err := quic.DialAddr(context.Background(), na.DialString(), tlsConf, &quickConfig) + conn, err := quic.DialAddr(ctx, na.DialString(), tlsConf, &quickConfig) if err != nil { return nil, err } diff --git a/p2p/peer.go b/p2p/peer.go index be9e73bd5a..c02869341d 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -545,13 +545,13 @@ func (p *peer) StartReceiving() error { for { stream, err := p.conn.AcceptStream(context.Background()) if err != nil { - p.Logger.Error("failed to accept stream", "err", err.Error()) + p.Logger.Debug("failed to accept stream", "err", err.Error()) return err } var chID byte err = binary.Read(stream, binary.BigEndian, &chID) if err != nil { - p.Logger.Error("failed to read channel ID", "err", err.Error()) + p.Logger.Debug("failed to read channel ID", "err", err.Error()) return err } // start accepting data @@ -560,13 +560,13 @@ func (p *peer) StartReceiving() error { var dataLen uint32 err = binary.Read(stream, binary.BigEndian, &dataLen) if err != nil { - p.Logger.Error("failed to read size from stream", "err", err.Error()) + p.Logger.Debug("failed to read size from stream", "err", err.Error()) return } data := make([]byte, dataLen) _, err = io.ReadFull(stream, data) if err != nil { - p.Logger.Error("failed to read data from stream", "err", err.Error()) + p.Logger.Debug("failed to read data from stream", "err", err.Error()) return } p.onReceive(chID, data) diff --git a/p2p/transport.go b/p2p/transport.go index 37628c0936..59c36c75c6 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -91,8 +91,10 @@ type ConnFilterFunc func(ConnSet, quic.Connection, []net.IP) error // and refuses new ones if they come from a known ip. func ConnDuplicateIPFilter() ConnFilterFunc { return func(cs ConnSet, c quic.Connection, ips []net.IP) error { + fmt.Println("filtering connections:") for _, ip := range ips { if cs.HasIP(ip) { + fmt.Println("rejected") return ErrRejected{ conn: c, err: fmt.Errorf("ip<%v> already connected", ip), @@ -244,7 +246,7 @@ func (mt *MultiplexTransport) Dial( return nil } - c, err := addr.DialTimeout(tlsConfig) + c, err := addr.DialTimeout(mt.dialTimeout, tlsConfig) if err != nil { return nil, err } @@ -254,14 +256,14 @@ func (mt *MultiplexTransport) Dial( return nil, err } - secretConn, nodeInfo, err := mt.getNodeInfo(c) + _, nodeInfo, err := mt.getNodeInfo(c) if err != nil { return nil, err } cfg.outbound = true - p := mt.wrapPeer(secretConn, nodeInfo, cfg, &addr) + p := mt.wrapPeer(c, nodeInfo, cfg, &addr) return p, nil } @@ -308,7 +310,7 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { quickConfig := quic.Config{ // TODO(rach-id): do we want to enable 0RTT? are the replay risks fine? Allow0RTT: false, - MaxIdleTimeout: 5 * time.Second, + MaxIdleTimeout: 200 * time.Millisecond, MaxIncomingStreams: 10000, MaxIncomingUniStreams: 10000, KeepAlivePeriod: 100 * time.Millisecond, @@ -392,8 +394,8 @@ func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { _, nodeInfo, err = mt.getNodeInfo(c) if err == nil { addr := c.RemoteAddr() - id := PubKeyToID(mt.nodeKey.PubKey()) - netAddr = NewNetAddress(id, addr) + //id := PubKeyToID(mt.nodeKey.PubKey()) + netAddr = NewNetAddress(nodeInfo.ID(), addr) } } @@ -402,8 +404,7 @@ func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { // Make the upgraded peer available. case <-mt.closec: // Give up if the transport was closed. - // TODO(rach-id): valid error - _ = c.CloseWithError(quic.ApplicationErrorCode(1), "some error 2") + _ = c.CloseWithError(quic.ApplicationErrorCode(1), "closes transport") return } }(c) @@ -428,7 +429,7 @@ func (mt *MultiplexTransport) filterConn(c quic.Connection) (err error) { defer func() { if err != nil { // TODO(rach-id): valid error - _ = c.CloseWithError(quic.ApplicationErrorCode(1), "some error 4") + _ = c.CloseWithError(quic.ApplicationErrorCode(1), err.Error()) } }() @@ -527,8 +528,11 @@ func exchangeNodeInfo( ourNodeInfo = nodeInfo.(DefaultNodeInfo) ) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + go func(errc chan<- error, c quic.Connection) { - stream, err := c.OpenStreamSync(context.Background()) + stream, err := c.OpenStreamSync(ctx) if err != nil { errc <- err return @@ -537,7 +541,7 @@ func exchangeNodeInfo( errc <- err }(errc, c) go func(errc chan<- error, c quic.Connection) { - stream, err := c.AcceptStream(context.Background()) + stream, err := c.AcceptStream(ctx) if err != nil { errc <- err return diff --git a/test/maverick/node/node.go b/test/maverick/node/node.go index bcc198c668..136b4007c1 100644 --- a/test/maverick/node/node.go +++ b/test/maverick/node/node.go @@ -18,7 +18,6 @@ import ( dbm "github.com/cometbft/cometbft-db" - abci "github.com/tendermint/tendermint/abci/types" bcv0 "github.com/tendermint/tendermint/blockchain/v0" bcv1 "github.com/tendermint/tendermint/blockchain/v1" bcv2 "github.com/tendermint/tendermint/blockchain/v2" @@ -550,55 +549,54 @@ func createTransport( []p2p.PeerFilterFunc, ) { var ( - mConnConfig = p2p.MConnConfig(config.P2P) - transport = p2p.NewMultiplexTransport(nodeInfo, *nodeKey, mConnConfig, trace.NoOpTracer()) + transport = p2p.NewMultiplexTransport(nodeInfo, *nodeKey, trace.NoOpTracer()) connFilters = []p2p.ConnFilterFunc{} peerFilters = []p2p.PeerFilterFunc{} ) - if !config.P2P.AllowDuplicateIP { - connFilters = append(connFilters, p2p.ConnDuplicateIPFilter()) - } - - // Filter peers by addr or pubkey with an ABCI query. - // If the query return code is OK, add peer. - if config.FilterPeers { - connFilters = append( - connFilters, - // ABCI query for address filtering. - func(_ p2p.ConnSet, c net.Conn, _ []net.IP) error { - res, err := proxyApp.Query().QuerySync(abci.RequestQuery{ - Path: fmt.Sprintf("/p2p/filter/addr/%s", c.RemoteAddr().String()), - }) - if err != nil { - return err - } - if res.IsErr() { - return fmt.Errorf("error querying abci app: %v", res) - } - - return nil - }, - ) - - peerFilters = append( - peerFilters, - // ABCI query for ID filtering. - func(_ p2p.IPeerSet, p p2p.Peer) error { - res, err := proxyApp.Query().QuerySync(abci.RequestQuery{ - Path: fmt.Sprintf("/p2p/filter/id/%s", p.ID()), - }) - if err != nil { - return err - } - if res.IsErr() { - return fmt.Errorf("error querying abci app: %v", res) - } - - return nil - }, - ) - } + //if !config.P2P.AllowDuplicateIP { + // connFilters = append(connFilters, p2p.ConnDuplicateIPFilter()) + //} + // + //// Filter peers by addr or pubkey with an ABCI query. + //// If the query return code is OK, add peer. + //if config.FilterPeers { + // connFilters = append( + // connFilters, + // // ABCI query for address filtering. + // func(_ p2p.ConnSet, c quic.Connection, _ []net.IP) error { + // res, err := proxyApp.Query().QuerySync(abci.RequestQuery{ + // Path: fmt.Sprintf("/p2p/filter/addr/%s", c.RemoteAddr().String()), + // }) + // if err != nil { + // return err + // } + // if res.IsErr() { + // return fmt.Errorf("error querying abci app: %v", res) + // } + // + // return nil + // }, + // ) + // + // peerFilters = append( + // peerFilters, + // // ABCI query for ID filtering. + // func(_ p2p.IPeerSet, p p2p.Peer) error { + // res, err := proxyApp.Query().QuerySync(abci.RequestQuery{ + // Path: fmt.Sprintf("/p2p/filter/id/%s", p.ID()), + // }) + // if err != nil { + // return err + // } + // if res.IsErr() { + // return fmt.Errorf("error querying abci app: %v", res) + // } + // + // return nil + // }, + // ) + //} p2p.MultiplexTransportConnFilters(connFilters...)(transport) From 196919f56788e48225f9972f981d1a33162406c0 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Thu, 29 Aug 2024 21:32:13 +0400 Subject: [PATCH 19/55] running e2e --- libs/net/net.go | 8 ++-- p2p/conn_set.go | 6 --- p2p/netaddress.go | 23 ++++++++++ p2p/peer.go | 10 ++--- p2p/transport.go | 14 +++--- test/maverick/node/node.go | 88 +++++++++++++++++++------------------- 6 files changed, 83 insertions(+), 66 deletions(-) diff --git a/libs/net/net.go b/libs/net/net.go index bed7d73853..fa85256fa8 100644 --- a/libs/net/net.go +++ b/libs/net/net.go @@ -17,10 +17,10 @@ func Connect(protoAddr string) (net.Conn, error) { // For instance, "tcp://127.0.0.1:8080" will be split into "tcp" and "127.0.0.1:8080". // If the address has no protocol prefix, the default is "tcp". func ProtocolAndAddress(listenAddr string) (string, string) { - protocol, address := "udp", listenAddr + protocol, address := "tcp", listenAddr parts := strings.SplitN(address, "://", 2) if len(parts) == 2 { - address = parts[1] + protocol, address = parts[0], parts[1] } return protocol, address } @@ -29,12 +29,12 @@ func ProtocolAndAddress(listenAddr string) (string, string) { // Ripped from https://github.com/phayes/freeport. // BSD-licensed. func GetFreePort() (int, error) { - addr, err := net.ResolveTCPAddr("udp", "localhost:0") + addr, err := net.ResolveTCPAddr("tcp", "localhost:0") if err != nil { return 0, err } - l, err := net.ListenTCP("udp", addr) + l, err := net.ListenTCP("tcp", addr) if err != nil { return 0, err } diff --git a/p2p/conn_set.go b/p2p/conn_set.go index 4f04568080..1fa5547540 100644 --- a/p2p/conn_set.go +++ b/p2p/conn_set.go @@ -1,7 +1,6 @@ package p2p import ( - "fmt" "github.com/quic-go/quic-go" "net" @@ -39,11 +38,6 @@ func (cs *connSet) Has(c quic.Connection) bool { cs.RLock() defer cs.RUnlock() - fmt.Println("has") - for key, val := range cs.conns { - fmt.Printf("%s:%v\n", key, val.ips) - } - fmt.Printf("looking for %s\n", c.RemoteAddr().String()) _, ok := cs.conns[c.RemoteAddr().String()] return ok diff --git a/p2p/netaddress.go b/p2p/netaddress.go index 98e5df00a5..d0925f79e5 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -44,6 +44,29 @@ func IDAddressString(id ID, protocolHostPort string) string { // panic. Panics if ID is invalid. // TODO: socks proxies? func NewNetAddress(id ID, addr net.Addr) *NetAddress { + tcpAddr, ok := addr.(*net.TCPAddr) + if !ok { + if flag.Lookup("test.v") == nil { // normal run + panic(fmt.Sprintf("Only TCPAddrs are supported. Got: %v", addr)) + } else { // in testing + netAddr := NewNetAddressIPPort(net.IP("127.0.0.1"), 0) + netAddr.ID = id + return netAddr + } + } + + if err := validateID(id); err != nil { + panic(fmt.Sprintf("Invalid ID %v: %v (addr: %v)", id, err, addr)) + } + + ip := tcpAddr.IP + port := uint16(tcpAddr.Port) + na := NewNetAddressIPPort(ip, port) + na.ID = id + return na +} + +func NewUDPNetAddress(id ID, addr net.Addr) *NetAddress { tcpAddr, ok := addr.(*net.UDPAddr) if !ok { if flag.Lookup("test.v") == nil { // normal run diff --git a/p2p/peer.go b/p2p/peer.go index c02869341d..58b5e32f2f 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -312,8 +312,8 @@ func (p *peer) FlushStop() { p.BaseService.OnStop() for _, stream := range p.streams { // TODO(rach-id): set valid error codes - stream.CancelRead(quic.StreamErrorCode(1)) // stop everything and close the conn - stream.CancelWrite(quic.StreamErrorCode(1)) // stop everything and close the conn + stream.CancelRead(quic.StreamErrorCode(0)) // stop everything and close the conn + stream.CancelWrite(quic.StreamErrorCode(0)) // stop everything and close the conn } } @@ -322,7 +322,7 @@ func (p *peer) OnStop() { p.metricsTicker.Stop() p.BaseService.OnStop() // TODO(rach-id): set valid error code - if err := p.conn.CloseWithError(quic.ApplicationErrorCode(2), "stopping peer connection"); err != nil { // stop everything and close the conn + if err := p.conn.CloseWithError(quic.ApplicationErrorCode(0), "stopping peer connection"); err != nil { // stop everything and close the conn p.Logger.Debug("Error while stopping peer", "err", err) } } @@ -496,7 +496,7 @@ func (p *peer) hasChannel(chID byte) bool { // CloseConn closes original connection. Used for cleaning up in cases where the peer had not been started at all. func (p *peer) CloseConn() error { // TODO(rach-id): valid error code - return p.conn.CloseWithError(quic.ApplicationErrorCode(1), "closed peer connection") + return p.conn.CloseWithError(quic.ApplicationErrorCode(0), "closed peer connection") } func (p *peer) SetRemovalFailed() { @@ -514,7 +514,7 @@ func (p *peer) GetRemovalFailed() bool { // CloseConn closes the underlying connection func (pc *peerConn) CloseConn() { // TODO(rach-id): valid error code - pc.conn.CloseWithError(quic.ApplicationErrorCode(1), "closed peer connection") + pc.conn.CloseWithError(quic.ApplicationErrorCode(0), "closed peer connection") } // RemoteAddr returns peer's remote network address. diff --git a/p2p/transport.go b/p2p/transport.go index 59c36c75c6..44d7775a41 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -91,10 +91,8 @@ type ConnFilterFunc func(ConnSet, quic.Connection, []net.IP) error // and refuses new ones if they come from a known ip. func ConnDuplicateIPFilter() ConnFilterFunc { return func(cs ConnSet, c quic.Connection, ips []net.IP) error { - fmt.Println("filtering connections:") for _, ip := range ips { if cs.HasIP(ip) { - fmt.Println("rejected") return ErrRejected{ conn: c, err: fmt.Errorf("ip<%v> already connected", ip), @@ -378,7 +376,7 @@ func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { case <-mt.closec: // Give up if the transport was closed. // TODO(rach-id): valid error code - _ = c.CloseWithError(quic.ApplicationErrorCode(1), "some error 1") + _ = c.CloseWithError(quic.ApplicationErrorCode(0), "some error 1") return } } @@ -395,7 +393,7 @@ func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { if err == nil { addr := c.RemoteAddr() //id := PubKeyToID(mt.nodeKey.PubKey()) - netAddr = NewNetAddress(nodeInfo.ID(), addr) + netAddr = NewUDPNetAddress(nodeInfo.ID(), addr) } } @@ -404,7 +402,7 @@ func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { // Make the upgraded peer available. case <-mt.closec: // Give up if the transport was closed. - _ = c.CloseWithError(quic.ApplicationErrorCode(1), "closes transport") + _ = c.CloseWithError(quic.ApplicationErrorCode(0), "closes transport") return } }(c) @@ -422,14 +420,14 @@ func (mt *MultiplexTransport) cleanup(c quic.Connection) error { mt.conns.Remove(c) // TODO(rach-id): valid error - return c.CloseWithError(quic.ApplicationErrorCode(1), "some error 3") + return c.CloseWithError(quic.ApplicationErrorCode(0), "some error 3") } func (mt *MultiplexTransport) filterConn(c quic.Connection) (err error) { defer func() { if err != nil { // TODO(rach-id): valid error - _ = c.CloseWithError(quic.ApplicationErrorCode(1), err.Error()) + _ = c.CloseWithError(quic.ApplicationErrorCode(0), err.Error()) } }() @@ -496,7 +494,7 @@ func (mt *MultiplexTransport) getNodeInfo(c quic.Connection) (conn quic.Connecti // Reject self. if mt.nodeInfo.ID() == remoteNodeInfo.ID() { return nil, nil, ErrRejected{ - addr: *NewNetAddress(remoteNodeInfo.ID(), c.RemoteAddr()), + addr: *NewUDPNetAddress(remoteNodeInfo.ID(), c.RemoteAddr()), conn: c, id: remoteNodeInfo.ID(), isSelf: true, diff --git a/test/maverick/node/node.go b/test/maverick/node/node.go index 136b4007c1..bdf80eb295 100644 --- a/test/maverick/node/node.go +++ b/test/maverick/node/node.go @@ -5,6 +5,8 @@ import ( "context" "errors" "fmt" + "github.com/quic-go/quic-go" + abci "github.com/tendermint/tendermint/abci/types" "net" "net/http" _ "net/http/pprof" //nolint: gosec // securely exposed on separate, optional port @@ -554,49 +556,49 @@ func createTransport( peerFilters = []p2p.PeerFilterFunc{} ) - //if !config.P2P.AllowDuplicateIP { - // connFilters = append(connFilters, p2p.ConnDuplicateIPFilter()) - //} - // - //// Filter peers by addr or pubkey with an ABCI query. - //// If the query return code is OK, add peer. - //if config.FilterPeers { - // connFilters = append( - // connFilters, - // // ABCI query for address filtering. - // func(_ p2p.ConnSet, c quic.Connection, _ []net.IP) error { - // res, err := proxyApp.Query().QuerySync(abci.RequestQuery{ - // Path: fmt.Sprintf("/p2p/filter/addr/%s", c.RemoteAddr().String()), - // }) - // if err != nil { - // return err - // } - // if res.IsErr() { - // return fmt.Errorf("error querying abci app: %v", res) - // } - // - // return nil - // }, - // ) - // - // peerFilters = append( - // peerFilters, - // // ABCI query for ID filtering. - // func(_ p2p.IPeerSet, p p2p.Peer) error { - // res, err := proxyApp.Query().QuerySync(abci.RequestQuery{ - // Path: fmt.Sprintf("/p2p/filter/id/%s", p.ID()), - // }) - // if err != nil { - // return err - // } - // if res.IsErr() { - // return fmt.Errorf("error querying abci app: %v", res) - // } - // - // return nil - // }, - // ) - //} + if !config.P2P.AllowDuplicateIP { + connFilters = append(connFilters, p2p.ConnDuplicateIPFilter()) + } + + // Filter peers by addr or pubkey with an ABCI query. + // If the query return code is OK, add peer. + if config.FilterPeers { + connFilters = append( + connFilters, + // ABCI query for address filtering. + func(_ p2p.ConnSet, c quic.Connection, _ []net.IP) error { + res, err := proxyApp.Query().QuerySync(abci.RequestQuery{ + Path: fmt.Sprintf("/p2p/filter/addr/%s", c.RemoteAddr().String()), + }) + if err != nil { + return err + } + if res.IsErr() { + return fmt.Errorf("error querying abci app: %v", res) + } + + return nil + }, + ) + + peerFilters = append( + peerFilters, + // ABCI query for ID filtering. + func(_ p2p.IPeerSet, p p2p.Peer) error { + res, err := proxyApp.Query().QuerySync(abci.RequestQuery{ + Path: fmt.Sprintf("/p2p/filter/id/%s", p.ID()), + }) + if err != nil { + return err + } + if res.IsErr() { + return fmt.Errorf("error querying abci app: %v", res) + } + + return nil + }, + ) + } p2p.MultiplexTransportConnFilters(connFilters...)(transport) From 0a5aebf81f7bc52dd8637f9b2ca35682ffb5abdd Mon Sep 17 00:00:00 2001 From: sweexordious Date: Fri, 20 Sep 2024 10:44:37 +0400 Subject: [PATCH 20/55] chore: merge v0.34.x --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 355ebc560d..d46ade14ab 100644 --- a/go.mod +++ b/go.mod @@ -138,7 +138,7 @@ require ( github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/go-toolsmith/astcast v1.1.0 // indirect github.com/go-toolsmith/astcopy v1.1.0 // indirect github.com/go-toolsmith/astequal v1.2.0 // indirect @@ -210,7 +210,6 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mgechev/revive v1.3.9 // indirect github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -223,7 +222,7 @@ require ( github.com/nishanths/predeclared v0.2.2 // indirect github.com/nunnatsa/ginkgolinter v0.16.2 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/onsi/ginkgo/v2 v2.9.5 // indirect + github.com/onsi/ginkgo/v2 v2.20.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc2 // indirect github.com/opencontainers/runc v1.1.3 // indirect @@ -234,8 +233,8 @@ require ( github.com/pkg/profile v1.7.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/polyfloyd/go-errorlint v1.6.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/quasilyte/go-ruleguard v0.4.3-0.20240823090925-0fe6f58b47b1 // indirect github.com/quasilyte/go-ruleguard/dsl v0.3.22 // indirect @@ -291,6 +290,7 @@ require ( go.opentelemetry.io/otel/trace v1.30.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/automaxprocs v1.5.3 // indirect + go.uber.org/mock v0.4.0 // indirect go.uber.org/multierr v1.10.0 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e // indirect diff --git a/go.sum b/go.sum index 63a5828fe5..d1db368ff4 100644 --- a/go.sum +++ b/go.sum @@ -332,7 +332,6 @@ github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -530,8 +529,6 @@ github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/Qd github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mgechev/revive v1.3.9 h1:18Y3R4a2USSBF+QZKFQwVkBROUda7uoBlkEuBD+YD1A= github.com/mgechev/revive v1.3.9/go.mod h1:+uxEIr5UH0TjXWHTno3xh4u7eg6jDpXKzQccA9UGhHU= github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM= @@ -620,12 +617,12 @@ github.com/polyfloyd/go-errorlint v1.6.0 h1:tftWV9DE7txiFzPpztTAwyoRLKNj9gpVm2cg github.com/polyfloyd/go-errorlint v1.6.0/go.mod h1:HR7u8wuP1kb1NeN1zqTd1ZMlqUKPPHF+Id4vIPvDqVw= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/quasilyte/go-ruleguard v0.4.3-0.20240823090925-0fe6f58b47b1 h1:+Wl/0aFp0hpuHM3H//KMft64WQ1yX9LdJY64Qm/gFCo= @@ -638,6 +635,8 @@ github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 h1:TCg2WBOl github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= +github.com/quic-go/quic-go v0.46.0 h1:uuwLClEEyk1DNvchH8uCByQVjo3yKL9opKulExNDs7Y= +github.com/quic-go/quic-go v0.46.0/go.mod h1:1dLehS7TIR64+vxGR70GDcatWTOtMX2PUtnKsjbTurI= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= @@ -835,6 +834,8 @@ go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= @@ -896,7 +897,6 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 4ace628e2cee79cd8ec93b17a89ea7108389315b Mon Sep 17 00:00:00 2001 From: sweexordious Date: Fri, 20 Sep 2024 22:23:20 +0400 Subject: [PATCH 21/55] chore: increase quic params --- p2p/netaddress.go | 8 ++++---- p2p/transport.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/p2p/netaddress.go b/p2p/netaddress.go index c0d1c71103..1488d8cffc 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -268,10 +268,10 @@ func (na *NetAddress) Dial(ctx context.Context) (quic.Connection, error) { InsecureSkipVerify: true, } quickConfig := quic.Config{ - MaxIdleTimeout: 5 * time.Second, + MaxIdleTimeout: 10 * time.Minute, MaxIncomingStreams: 10000, MaxIncomingUniStreams: 10000, - KeepAlivePeriod: time.Second, + KeepAlivePeriod: 2 * time.Minute, EnableDatagrams: true, } conn, err := quic.DialAddr(ctx, na.DialString(), &tlsConfig, &quickConfig) @@ -286,10 +286,10 @@ func (na *NetAddress) DialTimeout(timeout time.Duration, tlsConf *tls.Config) (q ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() quickConfig := quic.Config{ - MaxIdleTimeout: 200 * time.Millisecond, + MaxIdleTimeout: 10 * time.Minute, MaxIncomingStreams: 10000, MaxIncomingUniStreams: 10000, - KeepAlivePeriod: 100 * time.Millisecond, + KeepAlivePeriod: 2 * time.Minute, EnableDatagrams: true, } conn, err := quic.DialAddr(ctx, na.DialString(), tlsConf, &quickConfig) diff --git a/p2p/transport.go b/p2p/transport.go index 44d7775a41..5e4502a839 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -308,10 +308,10 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { quickConfig := quic.Config{ // TODO(rach-id): do we want to enable 0RTT? are the replay risks fine? Allow0RTT: false, - MaxIdleTimeout: 200 * time.Millisecond, + MaxIdleTimeout: 10 * time.Minute, MaxIncomingStreams: 10000, MaxIncomingUniStreams: 10000, - KeepAlivePeriod: 100 * time.Millisecond, + KeepAlivePeriod: 2 * time.Minute, EnableDatagrams: true, } listener, err := quic.ListenAddr(addr.DialString(), tlsConfig, &quickConfig) From 21d8e3825d7d6a468f67e7ada9be49d75ad9bdac Mon Sep 17 00:00:00 2001 From: sweexordious Date: Fri, 20 Sep 2024 22:23:36 +0400 Subject: [PATCH 22/55] fix: synchronisation when adding a new stream --- p2p/peer.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/p2p/peer.go b/p2p/peer.go index c77a21d224..1172059bfe 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -9,6 +9,7 @@ import ( "github.com/tendermint/tendermint/pkg/trace/schema" "io" "net" + "sync" "time" "github.com/gogo/protobuf/proto" @@ -166,6 +167,7 @@ func (pc peerConn) RemoteIP() net.IP { // // Before using a peer, you will need to perform a handshake on connection. type peer struct { + sync.Mutex service.BaseService // raw peerConn and the multiplex connection @@ -414,6 +416,12 @@ func (p *peer) SendEnvelope(e Envelope) bool { return res } +func (p *peer) addStream(stream quic.Stream, chID byte) { + p.Mutex.Lock() + defer p.Mutex.Unlock() + p.streams[chID] = stream +} + // Send msg bytes to the channel identified by chID byte. Returns false if the // send queue is full after timeout, specified by MConnection. // SendEnvelope replaces Send which will be deprecated in a future release. @@ -430,7 +438,7 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { p.Logger.Error("error opening quic stream", "err", err.Error()) return false } - p.streams[chID] = newStream + p.addStream(newStream, chID) stream = newStream err = binary.Write(stream, binary.BigEndian, chID) if err != nil { From 50a74f54cc09a707e5aac0ea2c917f85df60db6b Mon Sep 17 00:00:00 2001 From: sweexordious Date: Sun, 22 Sep 2024 21:39:49 +0400 Subject: [PATCH 23/55] to be reverted: remove peers filter --- p2p/switch.go | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/p2p/switch.go b/p2p/switch.go index bcfa8dffde..a2b28c320a 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -827,28 +827,28 @@ func (sw *Switch) addOutboundPeerWithConfig( func (sw *Switch) filterPeer(p Peer) error { // Avoid duplicate - if sw.peers.Has(p.ID()) { - return ErrRejected{id: p.ID(), isDuplicate: true} - } - - errc := make(chan error, len(sw.peerFilters)) - - for _, f := range sw.peerFilters { - go func(f PeerFilterFunc, p Peer, errc chan<- error) { - errc <- f(sw.peers, p) - }(f, p, errc) - } - - for i := 0; i < cap(errc); i++ { - select { - case err := <-errc: - if err != nil { - return ErrRejected{id: p.ID(), err: err, isFiltered: true} - } - case <-time.After(sw.filterTimeout): - return ErrFilterTimeout{} - } - } + //if sw.peers.Has(p.ID()) { + // return ErrRejected{id: p.ID(), isDuplicate: true} + //} + // + //errc := make(chan error, len(sw.peerFilters)) + // + //for _, f := range sw.peerFilters { + // go func(f PeerFilterFunc, p Peer, errc chan<- error) { + // errc <- f(sw.peers, p) + // }(f, p, errc) + //} + // + //for i := 0; i < cap(errc); i++ { + // select { + // case err := <-errc: + // if err != nil { + // return ErrRejected{id: p.ID(), err: err, isFiltered: true} + // } + // case <-time.After(sw.filterTimeout): + // return ErrFilterTimeout{} + // } + //} return nil } From 1307d9289afb90a4ba54da7790362218375052fa Mon Sep 17 00:00:00 2001 From: sweexordious Date: Sun, 22 Sep 2024 22:35:48 +0400 Subject: [PATCH 24/55] to be reverted: more synchronised access to streams --- p2p/peer.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index 1172059bfe..c907625b04 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -314,11 +314,11 @@ func (p *peer) OnStart() error { func (p *peer) FlushStop() { p.metricsTicker.Stop() p.BaseService.OnStop() - for _, stream := range p.streams { - // TODO(rach-id): set valid error codes - stream.CancelRead(quic.StreamErrorCode(0)) // stop everything and close the conn - stream.CancelWrite(quic.StreamErrorCode(0)) // stop everything and close the conn - } + //for _, stream := range p.streams { + // // TODO(rach-id): set valid error codes + // stream.CancelRead(quic.StreamErrorCode(0)) // stop everything and close the conn + // stream.CancelWrite(quic.StreamErrorCode(0)) // stop everything and close the conn + //} } // OnStop implements BaseService. @@ -422,6 +422,13 @@ func (p *peer) addStream(stream quic.Stream, chID byte) { p.streams[chID] = stream } +func (p *peer) getStream(chID byte) (quic.Stream, bool) { + p.Mutex.Lock() + defer p.Mutex.Unlock() + stream, has := p.streams[chID] + return stream, has +} + // Send msg bytes to the channel identified by chID byte. Returns false if the // send queue is full after timeout, specified by MConnection. // SendEnvelope replaces Send which will be deprecated in a future release. @@ -431,7 +438,7 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { } else if !p.hasChannel(chID) { return false } - stream, has := p.streams[chID] + stream, has := p.getStream(chID) if !has { newStream, err := p.conn.OpenStreamSync(context.Background()) if err != nil { From 060595a499426ab2c461e1dd8d49fd38f62c44a0 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 23 Sep 2024 00:57:52 +0400 Subject: [PATCH 25/55] to be reverted: more logs --- p2p/peer.go | 92 ++++++++++++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 44 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index c907625b04..3f22ae5dfb 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -235,7 +235,7 @@ func newPeer( msg := proto.Clone(mt) err := proto.Unmarshal(msgBytes, msg) if err != nil { - p.Logger.Error("before panic", "msg", msg, "channel", chID, "type", mt, "bytes", hex.EncodeToString(msgBytes)) + p.Logger.Error("before panic", "msg", msg, "channel", chID, "type", mt, "bytes", hex.EncodeToString(msgBytes), "raw_bytes", msgBytes) return } @@ -429,49 +429,6 @@ func (p *peer) getStream(chID byte) (quic.Stream, bool) { return stream, has } -// Send msg bytes to the channel identified by chID byte. Returns false if the -// send queue is full after timeout, specified by MConnection. -// SendEnvelope replaces Send which will be deprecated in a future release. -func (p *peer) Send(chID byte, msgBytes []byte) bool { - if !p.IsRunning() { - return false - } else if !p.hasChannel(chID) { - return false - } - stream, has := p.getStream(chID) - if !has { - newStream, err := p.conn.OpenStreamSync(context.Background()) - if err != nil { - p.Logger.Error("error opening quic stream", "err", err.Error()) - return false - } - p.addStream(newStream, chID) - stream = newStream - err = binary.Write(stream, binary.BigEndian, chID) - if err != nil { - p.Logger.Error("error sending channel ID", "err", err.Error()) - return false - } - } - - if err := binary.Write(stream, binary.BigEndian, uint32(len(msgBytes))); err != nil { - p.Logger.Error("Send len failed", "err", err, "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) - return false - } - err := binary.Write(stream, binary.BigEndian, msgBytes) - if err != nil { - p.Logger.Info("Send failed", "channel", "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) - return false - } - labels := []string{ - "peer_id", string(p.ID()), - "chID", fmt.Sprintf("%#x", chID), - } - p.metrics.PeerSendBytesTotal.With(labels...).Add(float64(len(msgBytes))) - - return true -} - // TrySend msg bytes to the channel identified by chID byte. Immediately returns // false if the send queue is full. // TrySendEnvelope replaces TrySend which will be deprecated in a future release. @@ -570,6 +527,51 @@ func PeerMetrics(metrics *Metrics) PeerOption { func (p *peer) metricsReporter() { } +// Send msg bytes to the channel identified by chID byte. Returns false if the +// send queue is full after timeout, specified by MConnection. +// SendEnvelope replaces Send which will be deprecated in a future release. +func (p *peer) Send(chID byte, msgBytes []byte) bool { + if !p.IsRunning() { + return false + } else if !p.hasChannel(chID) { + return false + } + stream, has := p.getStream(chID) + if !has { + newStream, err := p.conn.OpenStreamSync(context.Background()) + if err != nil { + p.Logger.Error("error opening quic stream", "err", err.Error()) + return false + } + p.addStream(newStream, chID) + stream = newStream + err = binary.Write(stream, binary.BigEndian, chID) + if err != nil { + p.Logger.Error("error sending channel ID", "err", err.Error()) + return false + } + } + + if err := binary.Write(stream, binary.BigEndian, uint32(len(msgBytes))); err != nil { + p.Logger.Error("Send len failed", "err", err, "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) + return false + } + err := binary.Write(stream, binary.BigEndian, msgBytes) + p.Logger.Info("sent data_len", "len", len(msgBytes)) + if err != nil { + p.Logger.Info("Send failed", "channel", "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) + return false + } + p.Logger.Info("sent data", "data", hex.EncodeToString(msgBytes), "raw_data", msgBytes) + labels := []string{ + "peer_id", string(p.ID()), + "chID", fmt.Sprintf("%#x", chID), + } + p.metrics.PeerSendBytesTotal.With(labels...).Add(float64(len(msgBytes))) + + return true +} + func (p *peer) StartReceiving() error { for { stream, err := p.conn.AcceptStream(context.Background()) @@ -592,12 +594,14 @@ func (p *peer) StartReceiving() error { p.Logger.Debug("failed to read size from stream", "err", err.Error()) return } + p.Logger.Info("received data len", "len", dataLen) data := make([]byte, dataLen) _, err = io.ReadFull(stream, data) if err != nil { p.Logger.Debug("failed to read data from stream", "err", err.Error()) return } + p.Logger.Info("received data", "bytes", hex.EncodeToString(data), "raw_bytes", data) p.onReceive(chID, data) } }() From c8e964cc13f5468a7e4eaad6fc4bede1bf32f788 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 23 Sep 2024 01:04:24 +0400 Subject: [PATCH 26/55] to be reverted: more logs --- p2p/peer.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index 3f22ae5dfb..471ef54093 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -557,12 +557,12 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { return false } err := binary.Write(stream, binary.BigEndian, msgBytes) - p.Logger.Info("sent data_len", "len", len(msgBytes)) + p.Logger.Debug("sent data_len", "len", len(msgBytes)) if err != nil { p.Logger.Info("Send failed", "channel", "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) return false } - p.Logger.Info("sent data", "data", hex.EncodeToString(msgBytes), "raw_data", msgBytes) + p.Logger.Debug("sent data", "data", hex.EncodeToString(msgBytes)) labels := []string{ "peer_id", string(p.ID()), "chID", fmt.Sprintf("%#x", chID), @@ -594,14 +594,14 @@ func (p *peer) StartReceiving() error { p.Logger.Debug("failed to read size from stream", "err", err.Error()) return } - p.Logger.Info("received data len", "len", dataLen) + p.Logger.Debug("received data len", "len", dataLen) data := make([]byte, dataLen) _, err = io.ReadFull(stream, data) if err != nil { p.Logger.Debug("failed to read data from stream", "err", err.Error()) return } - p.Logger.Info("received data", "bytes", hex.EncodeToString(data), "raw_bytes", data) + p.Logger.Debug("received data", "bytes", hex.EncodeToString(data)) p.onReceive(chID, data) } }() From 30d09ac0ceb9dab5a75083d89902c3d2cbb1a90f Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 23 Sep 2024 12:38:52 +0400 Subject: [PATCH 27/55] fix: use the p2p packet and proto reader/writer instead of raw bytes --- p2p/peer.go | 55 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index 471ef54093..d5e20040ef 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -6,8 +6,10 @@ import ( "encoding/hex" "fmt" "github.com/quic-go/quic-go" + "github.com/tendermint/tendermint/libs/protoio" "github.com/tendermint/tendermint/pkg/trace/schema" - "io" + "github.com/tendermint/tendermint/proto/tendermint/p2p" + "math" "net" "sync" "time" @@ -552,11 +554,21 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { } } - if err := binary.Write(stream, binary.BigEndian, uint32(len(msgBytes))); err != nil { - p.Logger.Error("Send len failed", "err", err, "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) - return false - } - err := binary.Write(stream, binary.BigEndian, msgBytes) + //if err := binary.Write(stream, binary.BigEndian, uint32(len(msgBytes))); err != nil { + // p.Logger.Error("Send len failed", "err", err, "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) + // return false + //} + packet := p2p.Packet{ + Sum: &p2p.Packet_PacketMsg{ + PacketMsg: &p2p.PacketMsg{ + ChannelID: int32(chID), + EOF: true, + Data: msgBytes, + }, + }, + } + _, err := protoio.NewDelimitedWriter(stream).WriteMsg(&packet) + //err := binary.Write(stream, binary.BigEndian, msgBytes) p.Logger.Debug("sent data_len", "len", len(msgBytes)) if err != nil { p.Logger.Info("Send failed", "channel", "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) @@ -587,22 +599,31 @@ func (p *peer) StartReceiving() error { } // start accepting data go func() { + //reader := protoio.NewDelimitedReader(stream, math.MaxInt32) for { - var dataLen uint32 - err = binary.Read(stream, binary.BigEndian, &dataLen) - if err != nil { - p.Logger.Debug("failed to read size from stream", "err", err.Error()) - return - } - p.Logger.Debug("received data len", "len", dataLen) - data := make([]byte, dataLen) - _, err = io.ReadFull(stream, data) + //var dataLen uint32 + //err = binary.Read(stream, binary.BigEndian, &dataLen) + //if err != nil { + // p.Logger.Debug("failed to read size from stream", "err", err.Error()) + // return + //} + //p.Logger.Debug("received data len", "len", dataLen) + //data := make([]byte, dataLen) + //_, err = io.ReadFull(stream, data) + //if err != nil { + // p.Logger.Debug("failed to read data from stream", "err", err.Error()) + // return + //} + var packet p2p.Packet + _, err := protoio.NewDelimitedReader(stream, math.MaxInt32).ReadMsg(&packet) if err != nil { p.Logger.Debug("failed to read data from stream", "err", err.Error()) return } - p.Logger.Debug("received data", "bytes", hex.EncodeToString(data)) - p.onReceive(chID, data) + + dd := packet.Sum.(*p2p.Packet_PacketMsg) + p.Logger.Debug("received data", "bytes", hex.EncodeToString(dd.PacketMsg.Data)) + p.onReceive(chID, dd.PacketMsg.Data) } }() } From fa7a8d613ac7707b68f9ce8be12245e2c98039c2 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 23 Sep 2024 20:09:32 +0400 Subject: [PATCH 28/55] chore: correct logs --- p2p/peer.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index d5e20040ef..e808514175 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -571,7 +571,7 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { //err := binary.Write(stream, binary.BigEndian, msgBytes) p.Logger.Debug("sent data_len", "len", len(msgBytes)) if err != nil { - p.Logger.Info("Send failed", "channel", "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) + p.Logger.Error("Send failed", "channel", "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) return false } p.Logger.Debug("sent data", "data", hex.EncodeToString(msgBytes)) @@ -588,13 +588,13 @@ func (p *peer) StartReceiving() error { for { stream, err := p.conn.AcceptStream(context.Background()) if err != nil { - p.Logger.Debug("failed to accept stream", "err", err.Error()) + p.Logger.Error("failed to accept stream", "err", err.Error()) return err } var chID byte err = binary.Read(stream, binary.BigEndian, &chID) if err != nil { - p.Logger.Debug("failed to read channel ID", "err", err.Error()) + p.Logger.Error("failed to read channel ID", "err", err.Error()) return err } // start accepting data @@ -617,7 +617,7 @@ func (p *peer) StartReceiving() error { var packet p2p.Packet _, err := protoio.NewDelimitedReader(stream, math.MaxInt32).ReadMsg(&packet) if err != nil { - p.Logger.Debug("failed to read data from stream", "err", err.Error()) + p.Logger.Error("failed to read data from stream", "err", err.Error()) return } From 574cbb89cd4a834989d5539587d5ef9ba2e4d70f Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 23 Sep 2024 20:11:33 +0400 Subject: [PATCH 29/55] to be reverted: remove filtering of peers --- p2p/transport.go | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/p2p/transport.go b/p2p/transport.go index 5e4502a839..f66c16fe2b 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -93,11 +93,11 @@ func ConnDuplicateIPFilter() ConnFilterFunc { return func(cs ConnSet, c quic.Connection, ips []net.IP) error { for _, ip := range ips { if cs.HasIP(ip) { - return ErrRejected{ - conn: c, - err: fmt.Errorf("ip<%v> already connected", ip), - isDuplicate: true, - } + //return ErrRejected{ + // conn: c, + // err: fmt.Errorf("ip<%v> already connected", ip), + // isDuplicate: true, + //} } } @@ -366,19 +366,19 @@ func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { go func(c quic.Connection) { defer func() { if r := recover(); r != nil { - err := ErrRejected{ - conn: c, - err: fmt.Errorf("recovered from panic: %v", r), - isAuthFailure: true, - } - select { - case mt.acceptc <- accept{err: err}: - case <-mt.closec: - // Give up if the transport was closed. - // TODO(rach-id): valid error code - _ = c.CloseWithError(quic.ApplicationErrorCode(0), "some error 1") - return - } + //err := ErrRejected{ + // conn: c, + // err: fmt.Errorf("recovered from panic: %v", r), + // isAuthFailure: true, + //} + //select { + //case mt.acceptc <- accept{err: err}: + //case <-mt.closec: + // // Give up if the transport was closed. + // // TODO(rach-id): valid error code + // _ = c.CloseWithError(quic.ApplicationErrorCode(0), "some error 1") + // return + //} } }() @@ -433,7 +433,8 @@ func (mt *MultiplexTransport) filterConn(c quic.Connection) (err error) { // Reject if connection is already present. if mt.conns.Has(c) { - return ErrRejected{conn: c, isDuplicate: true} + //return ErrRejected{conn: c, isDuplicate: true} + return nil } // Resolve ips for incoming conn. @@ -454,7 +455,8 @@ func (mt *MultiplexTransport) filterConn(c quic.Connection) (err error) { select { case err := <-errc: if err != nil { - return ErrRejected{conn: c, err: err, isFiltered: true} + return nil + //return ErrRejected{conn: c, err: err, isFiltered: true} } case <-time.After(mt.filterTimeout): return ErrFilterTimeout{} From 0b5ff4e5ff38c71ac41ea71230445c96031610ca Mon Sep 17 00:00:00 2001 From: sweexordious Date: Tue, 24 Sep 2024 00:11:35 +0400 Subject: [PATCH 30/55] to be reverted: just some logging --- p2p/peer.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index e808514175..f7ef5d25b5 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -574,7 +574,7 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { p.Logger.Error("Send failed", "channel", "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) return false } - p.Logger.Debug("sent data", "data", hex.EncodeToString(msgBytes)) + p.Logger.Debug("sent data", "channel", chID) labels := []string{ "peer_id", string(p.ID()), "chID", fmt.Sprintf("%#x", chID), @@ -622,9 +622,25 @@ func (p *peer) StartReceiving() error { } dd := packet.Sum.(*p2p.Packet_PacketMsg) - p.Logger.Debug("received data", "bytes", hex.EncodeToString(dd.PacketMsg.Data)) + //p.Logger.Debug("received data", "bytes", hex.EncodeToString(someLogData(dd.PacketMsg.Data))) + p.Logger.Debug("received data", "channel", chID) p.onReceive(chID, dd.PacketMsg.Data) } }() } } + +func someLogData(data []byte) []byte { + beginning := data + numBytes := 10 + if len(data) > numBytes { + beginning = data[:numBytes] + } + + // Get the last `numBytes` bytes from the end + end := data + if len(data) > numBytes { + end = data[len(data)-numBytes:] + } + return append(beginning, end...) +} From e976795b7b37d3e95e380befbc87410d5786d69b Mon Sep 17 00:00:00 2001 From: sweexordious Date: Fri, 11 Oct 2024 13:24:23 +0400 Subject: [PATCH 31/55] chore: update the idle timeout and keep alive period --- p2p/netaddress.go | 40 ++++++++++++++++++++-------------------- p2p/transport.go | 4 ++-- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/p2p/netaddress.go b/p2p/netaddress.go index 1488d8cffc..fa097dc857 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -262,34 +262,34 @@ func (na *NetAddress) DialString() string { // Dial calls net.Dial on the address. // TODO: add TLS stuff. // Note: this one is not used in the code, the DialTimeout is used instead. -func (na *NetAddress) Dial(ctx context.Context) (quic.Connection, error) { - tlsConfig := tls.Config{ - MinVersion: tls.VersionTLS13, - InsecureSkipVerify: true, - } - quickConfig := quic.Config{ - MaxIdleTimeout: 10 * time.Minute, - MaxIncomingStreams: 10000, - MaxIncomingUniStreams: 10000, - KeepAlivePeriod: 2 * time.Minute, - EnableDatagrams: true, - } - conn, err := quic.DialAddr(ctx, na.DialString(), &tlsConfig, &quickConfig) - if err != nil { - return nil, err - } - return conn, nil -} +//func (na *NetAddress) Dial(ctx context.Context) (quic.Connection, error) { +// tlsConfig := tls.Config{ +// MinVersion: tls.VersionTLS13, +// InsecureSkipVerify: true, +// } +// quickConfig := quic.Config{ +// MaxIdleTimeout: 10 * time.Minute, +// MaxIncomingStreams: 10000, +// MaxIncomingUniStreams: 10000, +// KeepAlivePeriod: 2 * time.Minute, +// EnableDatagrams: true, +// } +// conn, err := quic.DialAddr(ctx, na.DialString(), &tlsConfig, &quickConfig) +// if err != nil { +// return nil, err +// } +// return conn, nil +//} // DialTimeout calls net.DialTimeout on the address. func (na *NetAddress) DialTimeout(timeout time.Duration, tlsConf *tls.Config) (quic.Connection, error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() quickConfig := quic.Config{ - MaxIdleTimeout: 10 * time.Minute, + MaxIdleTimeout: time.Minute, MaxIncomingStreams: 10000, MaxIncomingUniStreams: 10000, - KeepAlivePeriod: 2 * time.Minute, + KeepAlivePeriod: 5 * time.Second, EnableDatagrams: true, } conn, err := quic.DialAddr(ctx, na.DialString(), tlsConf, &quickConfig) diff --git a/p2p/transport.go b/p2p/transport.go index f66c16fe2b..09da708f94 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -308,10 +308,10 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { quickConfig := quic.Config{ // TODO(rach-id): do we want to enable 0RTT? are the replay risks fine? Allow0RTT: false, - MaxIdleTimeout: 10 * time.Minute, + MaxIdleTimeout: time.Minute, MaxIncomingStreams: 10000, MaxIncomingUniStreams: 10000, - KeepAlivePeriod: 2 * time.Minute, + KeepAlivePeriod: 5 * time.Second, EnableDatagrams: true, } listener, err := quic.ListenAddr(addr.DialString(), tlsConfig, &quickConfig) From 2a7de97d134f9e9b087cf7a3edab51ab227246f7 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Sat, 12 Oct 2024 00:27:21 +0400 Subject: [PATCH 32/55] chore: revert previously commented code --- p2p/peer.go | 33 ++++++++++++++++++++++++-------- p2p/switch.go | 44 +++++++++++++++++++++---------------------- p2p/transport.go | 49 ++++++++++++++++++++++++------------------------ 3 files changed, 71 insertions(+), 55 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index f7ef5d25b5..8a9dbde1d0 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -316,11 +316,11 @@ func (p *peer) OnStart() error { func (p *peer) FlushStop() { p.metricsTicker.Stop() p.BaseService.OnStop() - //for _, stream := range p.streams { - // // TODO(rach-id): set valid error codes - // stream.CancelRead(quic.StreamErrorCode(0)) // stop everything and close the conn - // stream.CancelWrite(quic.StreamErrorCode(0)) // stop everything and close the conn - //} + for _, stream := range p.streams { + // TODO(rach-id): set valid error codes + stream.CancelRead(quic.StreamErrorCode(quic.NoError)) // stop everything and close the conn + stream.CancelWrite(quic.StreamErrorCode(quic.NoError)) // stop everything and close the conn + } } // OnStop implements BaseService. @@ -328,7 +328,7 @@ func (p *peer) OnStop() { p.metricsTicker.Stop() p.BaseService.OnStop() // TODO(rach-id): set valid error code - if err := p.conn.CloseWithError(quic.ApplicationErrorCode(0), "stopping peer connection"); err != nil { // stop everything and close the conn + if err := p.conn.CloseWithError(quic.ApplicationErrorCode(quic.NoError), "stopping peer connection onStop"); err != nil { // stop everything and close the conn p.Logger.Debug("Error while stopping peer", "err", err) } } @@ -484,7 +484,7 @@ func (p *peer) hasChannel(chID byte) bool { // CloseConn closes original connection. Used for cleaning up in cases where the peer had not been started at all. func (p *peer) CloseConn() error { // TODO(rach-id): valid error code - return p.conn.CloseWithError(quic.ApplicationErrorCode(0), "closed peer connection") + return p.conn.CloseWithError(quic.ApplicationErrorCode(quic.NoError), "closed peer connection closeCon") } func (p *peer) SetRemovalFailed() { @@ -502,7 +502,7 @@ func (p *peer) GetRemovalFailed() bool { // CloseConn closes the underlying connection func (pc *peerConn) CloseConn() { // TODO(rach-id): valid error code - pc.conn.CloseWithError(quic.ApplicationErrorCode(0), "closed peer connection") + pc.conn.CloseWithError(quic.ApplicationErrorCode(quic.NoError), "closed peer connection closeCon") } // RemoteAddr returns peer's remote network address. @@ -527,6 +527,23 @@ func PeerMetrics(metrics *Metrics) PeerOption { } func (p *peer) metricsReporter() { + //for { + // select { + // case <-p.metricsTicker.C: + // status := p.mconn.Status() + // var sendQueueSize float64 + // queues := make(map[byte]int, len(status.Channels)) + // for _, chStatus := range status.Channels { + // sendQueueSize += float64(chStatus.SendQueueSize) + // queues[chStatus.ID] = chStatus.SendQueueSize + // } + // + // p.metrics.PeerPendingSendBytes.With("peer_id", string(p.ID())).Set(sendQueueSize) + // schema.WritePendingBytes(p.traceClient, string(p.ID()), queues) + // case <-p.Quit(): + // return + // } + //} } // Send msg bytes to the channel identified by chID byte. Returns false if the diff --git a/p2p/switch.go b/p2p/switch.go index a2b28c320a..bcfa8dffde 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -827,28 +827,28 @@ func (sw *Switch) addOutboundPeerWithConfig( func (sw *Switch) filterPeer(p Peer) error { // Avoid duplicate - //if sw.peers.Has(p.ID()) { - // return ErrRejected{id: p.ID(), isDuplicate: true} - //} - // - //errc := make(chan error, len(sw.peerFilters)) - // - //for _, f := range sw.peerFilters { - // go func(f PeerFilterFunc, p Peer, errc chan<- error) { - // errc <- f(sw.peers, p) - // }(f, p, errc) - //} - // - //for i := 0; i < cap(errc); i++ { - // select { - // case err := <-errc: - // if err != nil { - // return ErrRejected{id: p.ID(), err: err, isFiltered: true} - // } - // case <-time.After(sw.filterTimeout): - // return ErrFilterTimeout{} - // } - //} + if sw.peers.Has(p.ID()) { + return ErrRejected{id: p.ID(), isDuplicate: true} + } + + errc := make(chan error, len(sw.peerFilters)) + + for _, f := range sw.peerFilters { + go func(f PeerFilterFunc, p Peer, errc chan<- error) { + errc <- f(sw.peers, p) + }(f, p, errc) + } + + for i := 0; i < cap(errc); i++ { + select { + case err := <-errc: + if err != nil { + return ErrRejected{id: p.ID(), err: err, isFiltered: true} + } + case <-time.After(sw.filterTimeout): + return ErrFilterTimeout{} + } + } return nil } diff --git a/p2p/transport.go b/p2p/transport.go index 09da708f94..7715e1f37c 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "github.com/quic-go/quic-go" + "github.com/quic-go/quic-go/http3" "github.com/tendermint/tendermint/libs/protoio" tmp2p "github.com/tendermint/tendermint/proto/tendermint/p2p" "net" @@ -93,11 +94,11 @@ func ConnDuplicateIPFilter() ConnFilterFunc { return func(cs ConnSet, c quic.Connection, ips []net.IP) error { for _, ip := range ips { if cs.HasIP(ip) { - //return ErrRejected{ - // conn: c, - // err: fmt.Errorf("ip<%v> already connected", ip), - // isDuplicate: true, - //} + return ErrRejected{ + conn: c, + err: fmt.Errorf("ip<%v> already connected", ip), + isDuplicate: true, + } } } @@ -366,19 +367,19 @@ func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { go func(c quic.Connection) { defer func() { if r := recover(); r != nil { - //err := ErrRejected{ - // conn: c, - // err: fmt.Errorf("recovered from panic: %v", r), - // isAuthFailure: true, - //} - //select { - //case mt.acceptc <- accept{err: err}: - //case <-mt.closec: - // // Give up if the transport was closed. - // // TODO(rach-id): valid error code - // _ = c.CloseWithError(quic.ApplicationErrorCode(0), "some error 1") - // return - //} + err := ErrRejected{ + conn: c, + err: fmt.Errorf("recovered from panic: %v", r), + isAuthFailure: true, + } + select { + case mt.acceptc <- accept{err: err}: + case <-mt.closec: + // Give up if the transport was closed. + // TODO(rach-id): valid error code + _ = c.CloseWithError(quic.ApplicationErrorCode(http3.ErrCodeConnectError), err.Error()) + return + } } }() @@ -402,7 +403,7 @@ func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { // Make the upgraded peer available. case <-mt.closec: // Give up if the transport was closed. - _ = c.CloseWithError(quic.ApplicationErrorCode(0), "closes transport") + _ = c.CloseWithError(quic.ApplicationErrorCode(http3.ErrCodeConnectError), "closes transport") return } }(c) @@ -420,21 +421,20 @@ func (mt *MultiplexTransport) cleanup(c quic.Connection) error { mt.conns.Remove(c) // TODO(rach-id): valid error - return c.CloseWithError(quic.ApplicationErrorCode(0), "some error 3") + return c.CloseWithError(quic.ApplicationErrorCode(quic.NoError), "closing for cleanup") } func (mt *MultiplexTransport) filterConn(c quic.Connection) (err error) { defer func() { if err != nil { // TODO(rach-id): valid error - _ = c.CloseWithError(quic.ApplicationErrorCode(0), err.Error()) + _ = c.CloseWithError(quic.ApplicationErrorCode(http3.ErrCodeConnectError), err.Error()) } }() // Reject if connection is already present. if mt.conns.Has(c) { - //return ErrRejected{conn: c, isDuplicate: true} - return nil + return ErrRejected{conn: c, isDuplicate: true} } // Resolve ips for incoming conn. @@ -455,8 +455,7 @@ func (mt *MultiplexTransport) filterConn(c quic.Connection) (err error) { select { case err := <-errc: if err != nil { - return nil - //return ErrRejected{conn: c, err: err, isFiltered: true} + return ErrRejected{conn: c, err: err, isFiltered: true} } case <-time.After(mt.filterTimeout): return ErrFilterTimeout{} From f57f4bdaba9415a77fec38ff95122aa19ae93829 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 21 Oct 2024 13:12:28 +0400 Subject: [PATCH 33/55] chore: trying for better peer management --- blockchain/v0/reactor.go | 2 +- blockchain/v1/reactor.go | 2 +- blockchain/v2/reactor.go | 2 +- consensus/reactor.go | 2 +- go.mod | 1 + go.sum | 2 + mempool/v0/reactor.go | 4 +- mempool/v1/reactor.go | 4 +- p2p/netaddress.go | 6 ++- p2p/peer.go | 35 +++++++------ p2p/switch.go | 37 +++++++++++++- p2p/transport.go | 104 ++++++++++++++++++++++++++++++++++++++- 12 files changed, 174 insertions(+), 27 deletions(-) diff --git a/blockchain/v0/reactor.go b/blockchain/v0/reactor.go index 090cd6f760..ed6fa079f3 100644 --- a/blockchain/v0/reactor.go +++ b/blockchain/v0/reactor.go @@ -202,7 +202,7 @@ func (bcR *BlockchainReactor) ReceiveEnvelope(e p2p.Envelope) { return } - bcR.Logger.Debug("Receive", "e.Src", e.Src, "chID", e.ChannelID, "msg", e.Message) + //bcR.Logger.Debug("Receive", "e.Src", e.Src, "chID", e.ChannelID, "msg", e.Message) switch msg := e.Message.(type) { case *bcproto.BlockRequest: diff --git a/blockchain/v1/reactor.go b/blockchain/v1/reactor.go index cc342a3224..ae14aaf8d0 100644 --- a/blockchain/v1/reactor.go +++ b/blockchain/v1/reactor.go @@ -250,7 +250,7 @@ func (bcR *BlockchainReactor) ReceiveEnvelope(e p2p.Envelope) { return } - bcR.Logger.Debug("Receive", "src", e.Src, "chID", e.ChannelID, "msg", e.Message) + //bcR.Logger.Debug("Receive", "src", e.Src, "chID", e.ChannelID, "msg", e.Message) switch msg := e.Message.(type) { case *bcproto.BlockRequest: diff --git a/blockchain/v2/reactor.go b/blockchain/v2/reactor.go index 7c3330a528..349877e414 100644 --- a/blockchain/v2/reactor.go +++ b/blockchain/v2/reactor.go @@ -462,7 +462,7 @@ func (r *BlockchainReactor) ReceiveEnvelope(e p2p.Envelope) { return } - r.logger.Debug("Receive", "src", e.Src.ID(), "chID", e.ChannelID, "msg", e.Message) + //r.logger.Debug("Receive", "src", e.Src.ID(), "chID", e.ChannelID, "msg", e.Message) switch msg := e.Message.(type) { case *bcproto.StatusRequest: diff --git a/consensus/reactor.go b/consensus/reactor.go index 3e99824c2d..37feec35ed 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -257,7 +257,7 @@ func (conR *Reactor) ReceiveEnvelope(e p2p.Envelope) { return } - conR.Logger.Debug("Receive", "src", e.Src, "chId", e.ChannelID, "msg", msg) + //conR.Logger.Debug("Receive", "src", e.Src, "chId", e.ChannelID, "msg", msg) // Get peer states ps, ok := e.Src.Get(types.PeerStateKey).(*PeerState) diff --git a/go.mod b/go.mod index d46ade14ab..ff3dfba90a 100644 --- a/go.mod +++ b/go.mod @@ -241,6 +241,7 @@ require ( github.com/quasilyte/gogrep v0.5.0 // indirect github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect + github.com/quic-go/qpack v0.4.0 // indirect github.com/rs/zerolog v1.29.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/ryancurrah/gomodguard v1.3.5 // indirect diff --git a/go.sum b/go.sum index d1db368ff4..d5f80cb990 100644 --- a/go.sum +++ b/go.sum @@ -635,6 +635,8 @@ github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 h1:TCg2WBOl github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= +github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= +github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/quic-go v0.46.0 h1:uuwLClEEyk1DNvchH8uCByQVjo3yKL9opKulExNDs7Y= github.com/quic-go/quic-go v0.46.0/go.mod h1:1dLehS7TIR64+vxGR70GDcatWTOtMX2PUtnKsjbTurI= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= diff --git a/mempool/v0/reactor.go b/mempool/v0/reactor.go index 85b5298518..45b1f099c8 100644 --- a/mempool/v0/reactor.go +++ b/mempool/v0/reactor.go @@ -158,7 +158,7 @@ func (memR *Reactor) RemovePeer(peer p2p.Peer, reason interface{}) { // Receive implements Reactor. // It adds any received transactions to the mempool. func (memR *Reactor) ReceiveEnvelope(e p2p.Envelope) { - memR.Logger.Debug("Receive", "src", e.Src, "chId", e.ChannelID, "msg", e.Message) + //memR.Logger.Debug("Receive", "src", e.Src, "chId", e.ChannelID, "msg", e.Message) switch msg := e.Message.(type) { case *protomem.Txs: protoTxs := msg.GetTxs() @@ -176,7 +176,7 @@ func (memR *Reactor) ReceiveEnvelope(e p2p.Envelope) { ntx := types.Tx(tx) err = memR.mempool.CheckTx(ntx, nil, txInfo) if errors.Is(err, mempool.ErrTxInCache) { - memR.Logger.Debug("Tx already exists in cache", "tx", ntx.String()) + memR.Logger.Debug("Tx already exists in cache", "tx", ntx.Hash()) } else if err != nil { memR.Logger.Info("Could not check tx", "tx", ntx.String(), "err", err) } diff --git a/mempool/v1/reactor.go b/mempool/v1/reactor.go index 7e0b22bcf5..dd65ddede2 100644 --- a/mempool/v1/reactor.go +++ b/mempool/v1/reactor.go @@ -177,7 +177,7 @@ func (memR *Reactor) RemovePeer(peer p2p.Peer, reason interface{}) { // Receive implements Reactor. // It adds any received transactions to the mempool. func (memR *Reactor) ReceiveEnvelope(e p2p.Envelope) { - memR.Logger.Debug("Receive", "src", e.Src, "chId", e.ChannelID, "msg", e.Message) + //memR.Logger.Debug("Receive", "src", e.Src, "chId", e.ChannelID, "msg", e.Message) switch msg := e.Message.(type) { case *protomem.Txs: protoTxs := msg.GetTxs() @@ -202,7 +202,7 @@ func (memR *Reactor) ReceiveEnvelope(e p2p.Envelope) { ) err = memR.mempool.CheckTx(ntx, nil, txInfo) if errors.Is(err, mempool.ErrTxInCache) { - memR.Logger.Debug("Tx already exists in cache", "tx", ntx.String()) + memR.Logger.Debug("Tx already exists in cache", "tx", ntx.Hash()) } else if err != nil { memR.Logger.Info("Could not check tx", "tx", ntx.String(), "err", err) } diff --git a/p2p/netaddress.go b/p2p/netaddress.go index fa097dc857..c5c3bde6c2 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -12,6 +12,7 @@ import ( "flag" "fmt" "github.com/quic-go/quic-go" + "github.com/quic-go/quic-go/logging" "net" "strconv" "strings" @@ -289,8 +290,11 @@ func (na *NetAddress) DialTimeout(timeout time.Duration, tlsConf *tls.Config) (q MaxIdleTimeout: time.Minute, MaxIncomingStreams: 10000, MaxIncomingUniStreams: 10000, - KeepAlivePeriod: 5 * time.Second, + KeepAlivePeriod: 10 * time.Second, EnableDatagrams: true, + Tracer: func(ctx context.Context, perspective logging.Perspective, id quic.ConnectionID) *logging.ConnectionTracer { + return logging.NewMultiplexedConnectionTracer(GetNewTracer()) + }, } conn, err := quic.DialAddr(ctx, na.DialString(), tlsConf, &quickConfig) if err != nil { diff --git a/p2p/peer.go b/p2p/peer.go index 8a9dbde1d0..2fc659ed39 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -58,6 +58,8 @@ type Peer interface { SetRemovalFailed() GetRemovalFailed() bool + + GetConnectionContext() context.Context } type EnvelopeSender interface { @@ -424,6 +426,12 @@ func (p *peer) addStream(stream quic.Stream, chID byte) { p.streams[chID] = stream } +func (p *peer) removeStream(chID byte) { + p.Mutex.Lock() + defer p.Mutex.Unlock() + delete(p.streams, chID) +} + func (p *peer) getStream(chID byte) (quic.Stream, bool) { p.Mutex.Lock() defer p.Mutex.Unlock() @@ -616,21 +624,7 @@ func (p *peer) StartReceiving() error { } // start accepting data go func() { - //reader := protoio.NewDelimitedReader(stream, math.MaxInt32) for { - //var dataLen uint32 - //err = binary.Read(stream, binary.BigEndian, &dataLen) - //if err != nil { - // p.Logger.Debug("failed to read size from stream", "err", err.Error()) - // return - //} - //p.Logger.Debug("received data len", "len", dataLen) - //data := make([]byte, dataLen) - //_, err = io.ReadFull(stream, data) - //if err != nil { - // p.Logger.Debug("failed to read data from stream", "err", err.Error()) - // return - //} var packet p2p.Packet _, err := protoio.NewDelimitedReader(stream, math.MaxInt32).ReadMsg(&packet) if err != nil { @@ -639,7 +633,6 @@ func (p *peer) StartReceiving() error { } dd := packet.Sum.(*p2p.Packet_PacketMsg) - //p.Logger.Debug("received data", "bytes", hex.EncodeToString(someLogData(dd.PacketMsg.Data))) p.Logger.Debug("received data", "channel", chID) p.onReceive(chID, dd.PacketMsg.Data) } @@ -661,3 +654,15 @@ func someLogData(data []byte) []byte { } return append(beginning, end...) } + +func (p *peer) SendDatagram(bytes []byte) error { + return p.conn.SendDatagram(bytes) +} + +func (p *peer) ReceiveDatagram(ctx context.Context) ([]byte, error) { + return p.conn.ReceiveDatagram(ctx) +} + +func (p *peer) GetConnectionContext() context.Context { + return p.conn.Context() +} diff --git a/p2p/switch.go b/p2p/switch.go index bcfa8dffde..48b74dae00 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -121,6 +121,15 @@ func NewSwitch( traceClient: trace.NoOpTracer(), } + go func() { + for { + fmt.Println("=================================") + fmt.Println(sw.peers.list) + fmt.Println("=================================") + time.Sleep(5 * time.Second) + } + }() + // Ensure we have a completely undeterministic PRNG. sw.rng = rand.NewRand() @@ -363,7 +372,7 @@ func (sw *Switch) Peers() IPeerSet { // TODO: make record depending on reason. func (sw *Switch) StopPeerForError(peer Peer, reason interface{}) { if !peer.IsRunning() { - return + fmt.Println("Peer is not running!!!! we should stop but we CAN'T!!!!") } sw.Logger.Error("Stopping peer for error", "peer", peer, "err", reason) @@ -406,18 +415,21 @@ func (sw *Switch) getPeerAddress(peer Peer) (*NetAddress, error) { // StopPeerGracefully disconnects from a peer gracefully. // TODO: handle graceful disconnects. func (sw *Switch) StopPeerGracefully(peer Peer) { - sw.Logger.Info("Stopping peer gracefully") + sw.Logger.Info("Stopping peer gracefully", "peer", peer.ID()) sw.stopAndRemovePeer(peer, nil) } func (sw *Switch) stopAndRemovePeer(peer Peer, reason interface{}) { sw.transport.Cleanup(peer) schema.WritePeerUpdate(sw.traceClient, string(peer.ID()), schema.PeerDisconnect, fmt.Sprintf("%v", reason)) + sw.Logger.Error("entering stop and remove peer", "peer", peer.ID()) if err := peer.Stop(); err != nil { sw.Logger.Error("error while stopping peer", "error", err) // TODO: should return error to be handled accordingly } + sw.Logger.Error("after peer stopped", "peer", peer.ID()) for _, reactor := range sw.reactors { + sw.Logger.Error("stopping peer reactors", "peer", peer.ID(), "reactor", reactor) reactor.RemovePeer(peer, reason) } @@ -425,6 +437,7 @@ func (sw *Switch) stopAndRemovePeer(peer Peer, reason interface{}) { // reconnect to our node and the switch calls InitPeer before // RemovePeer is finished. // https://github.com/tendermint/tendermint/issues/3338 + sw.Logger.Error("after removing all reactors", "peer", peer.ID()) if sw.peers.Remove(peer) { sw.metrics.Peers.Add(float64(-1)) } else { @@ -432,6 +445,7 @@ func (sw *Switch) stopAndRemovePeer(peer Peer, reason interface{}) { // We keep this message here as information to the developer. sw.Logger.Debug("error on peer removal", ",", "peer", peer.ID()) } + sw.Logger.Error("after removing peer from peers list", "peer", peer.ID()) } // reconnectToPeer tries to reconnect to the addr, first repeatedly @@ -907,5 +921,24 @@ func (sw *Switch) addPeer(p Peer) error { sw.Logger.Debug("Added peer", "peer", p) + go func() { + fmt.Println("listening for connection to stop") + fmt.Println(p.ID()) + fmt.Println(p.RemoteAddr().String()) + fmt.Println("------------------------------------") + select { + case <-p.GetConnectionContext().Done(): + fmt.Println("----------------------------- connection stopped, context cancelled") + fmt.Println(p.ID()) + fmt.Println(p.RemoteAddr().String()) + fmt.Println("------------------------------------") + err := p.Stop() + if err != nil { + fmt.Println(err) + } + sw.StopPeerForError(p, p.GetConnectionContext().Err()) + } + }() + return nil } diff --git a/p2p/transport.go b/p2p/transport.go index 7715e1f37c..cfcb5e8dac 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/quic-go/quic-go" "github.com/quic-go/quic-go/http3" + "github.com/quic-go/quic-go/logging" "github.com/tendermint/tendermint/libs/protoio" tmp2p "github.com/tendermint/tendermint/proto/tendermint/p2p" "net" @@ -312,8 +313,11 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { MaxIdleTimeout: time.Minute, MaxIncomingStreams: 10000, MaxIncomingUniStreams: 10000, - KeepAlivePeriod: 5 * time.Second, + KeepAlivePeriod: 10 * time.Second, EnableDatagrams: true, + Tracer: func(ctx context.Context, perspective logging.Perspective, id quic.ConnectionID) *logging.ConnectionTracer { + return logging.NewMultiplexedConnectionTracer(GetNewTracer()) + }, } listener, err := quic.ListenAddr(addr.DialString(), tlsConfig, &quickConfig) if err != nil { @@ -328,6 +332,104 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { return nil } +func GetNewTracer() *logging.ConnectionTracer { + return &logging.ConnectionTracer{ + StartedConnection: func(local, remote net.Addr, srcConnID, destConnID quic.ConnectionID) { + fmt.Println(fmt.Sprintf("StartedConnection: local=%v, remote=%v, srcConnID=%v, destConnID=%v", local, remote, srcConnID, destConnID)) + }, + NegotiatedVersion: func(chosen quic.Version, clientVersions, serverVersions []quic.Version) { + //fmt.Println(fmt.Sprintf("NegotiatedVersion: chosen=%v, clientVersions=%v, serverVersions=%v", chosen, clientVersions, serverVersions)) + }, + ClosedConnection: func(err error) { + fmt.Println(fmt.Sprintf("ClosedConnection: error=%v", err)) + }, + SentTransportParameters: func(params *logging.TransportParameters) { + //fmt.Println(fmt.Sprintf("SentTransportParameters: params=%v", params)) + }, + ReceivedTransportParameters: func(params *logging.TransportParameters) { + //fmt.Println(fmt.Sprintf("ReceivedTransportParameters: params=%v", params)) + }, + RestoredTransportParameters: func(params *logging.TransportParameters) { + //fmt.Println(fmt.Sprintf("RestoredTransportParameters: params=%v", params)) + }, + SentLongHeaderPacket: func(header *logging.ExtendedHeader, byteCount logging.ByteCount, ecn logging.ECN, ack *logging.AckFrame, frames []logging.Frame) { + //fmt.Println(fmt.Sprintf("SentLongHeaderPacket: header=%v, byteCount=%v, ecn=%v, ack=%v, frames=%v", header, byteCount, ecn, ack, frames)) + }, + SentShortHeaderPacket: func(header *logging.ShortHeader, byteCount logging.ByteCount, ecn logging.ECN, ack *logging.AckFrame, frames []logging.Frame) { + //fmt.Println(fmt.Sprintf("SentShortHeaderPacket: header=%v, byteCount=%v, ecn=%v, ack=%v, frames=%v", header, byteCount, ecn, ack, frames)) + }, + ReceivedVersionNegotiationPacket: func(dest, src logging.ArbitraryLenConnectionID, versions []logging.Version) { + //fmt.Println(fmt.Sprintf("ReceivedVersionNegotiationPacket: dest=%v, src=%v, versions=%v", dest, src, versions)) + }, + ReceivedRetry: func(header *logging.Header) { + fmt.Println(fmt.Sprintf("ReceivedRetry: header=%v", header)) + }, + ReceivedLongHeaderPacket: func(header *logging.ExtendedHeader, byteCount logging.ByteCount, ecn logging.ECN, frames []logging.Frame) { + //fmt.Println(fmt.Sprintf("ReceivedLongHeaderPacket: header=%v, byteCount=%v, ecn=%v, frames=%v", header, byteCount, ecn, frames)) + }, + ReceivedShortHeaderPacket: func(header *logging.ShortHeader, byteCount logging.ByteCount, ecn logging.ECN, frames []logging.Frame) { + //fmt.Println(fmt.Sprintf("ReceivedShortHeaderPacket: header=%v, byteCount=%v, ecn=%v, frames=%v", header, byteCount, ecn, frames)) + }, + BufferedPacket: func(packetType logging.PacketType, byteCount logging.ByteCount) { + //fmt.Println(fmt.Sprintf("BufferedPacket: packetType=%v, byteCount=%v", packetType, byteCount)) + }, + DroppedPacket: func(packetType logging.PacketType, packetNum logging.PacketNumber, byteCount logging.ByteCount, reason logging.PacketDropReason) { + fmt.Println(fmt.Sprintf("DroppedPacket: packetType=%v, packetNum=%v, byteCount=%v, reason=%v", packetType, packetNum, byteCount, reason)) + }, + UpdatedMetrics: func(rttStats *logging.RTTStats, cwnd, bytesInFlight logging.ByteCount, packetsInFlight int) { + //fmt.Println(fmt.Sprintf("UpdatedMetrics: rttStats=%v, cwnd=%v, bytesInFlight=%v, packetsInFlight=%v", rttStats, cwnd, bytesInFlight, packetsInFlight)) + }, + AcknowledgedPacket: func(encLevel logging.EncryptionLevel, packetNum logging.PacketNumber) { + //fmt.Println(fmt.Sprintf("AcknowledgedPacket: encLevel=%v, packetNum=%v", encLevel, packetNum)) + }, + LostPacket: func(encLevel logging.EncryptionLevel, packetNum logging.PacketNumber, reason logging.PacketLossReason) { + fmt.Println(fmt.Sprintf("LostPacket: encLevel=%v, packetNum=%v, reason=%v", encLevel, packetNum, reason)) + }, + UpdatedMTU: func(mtu logging.ByteCount, done bool) { + //fmt.Println(fmt.Sprintf("UpdatedMTU: mtu=%v, done=%v", mtu, done)) + }, + UpdatedCongestionState: func(state logging.CongestionState) { + fmt.Println(fmt.Sprintf("UpdatedCongestionState: state=%v", state)) + }, + UpdatedPTOCount: func(value uint32) { + //fmt.Println(fmt.Sprintf("UpdatedPTOCount: value=%v", value)) + }, + UpdatedKeyFromTLS: func(encLevel logging.EncryptionLevel, perspective logging.Perspective) { + //fmt.Println(fmt.Sprintf("UpdatedKeyFromTLS: encLevel=%v, perspective=%v", encLevel, perspective)) + }, + UpdatedKey: func(phase logging.KeyPhase, remote bool) { + //fmt.Println(fmt.Sprintf("UpdatedKey: phase=%v, remote=%v", phase, remote)) + }, + DroppedEncryptionLevel: func(encLevel logging.EncryptionLevel) { + //fmt.Println(fmt.Sprintf("DroppedEncryptionLevel: encLevel=%v", encLevel)) + }, + DroppedKey: func(phase logging.KeyPhase) { + //fmt.Println(fmt.Sprintf("DroppedKey: phase=%v", phase)) + }, + SetLossTimer: func(timerType logging.TimerType, encLevel logging.EncryptionLevel, t time.Time) { + //fmt.Println(fmt.Sprintf("SetLossTimer: timerType=%v, encLevel=%v, time=%v", timerType, encLevel, t)) + }, + LossTimerExpired: func(timerType logging.TimerType, encLevel logging.EncryptionLevel) { + //fmt.Println(fmt.Sprintf("LossTimerExpired: timerType=%v, encLevel=%v", timerType, encLevel)) + }, + LossTimerCanceled: func() { + //fmt.Println(fmt.Sprintf("LossTimerCanceled")) + }, + ECNStateUpdated: func(state logging.ECNState, trigger logging.ECNStateTrigger) { + //fmt.Println(fmt.Sprintf("ECNStateUpdated: state=%v, trigger=%v", state, trigger)) + }, + ChoseALPN: func(protocol string) { + //fmt.Println(fmt.Sprintf("ChoseALPN: protocol=%v", protocol)) + }, + Close: func() { + fmt.Println(fmt.Sprintf("Close")) + }, + Debug: func(name, msg string) { + fmt.Println(fmt.Sprintf("Debug: name=%v, msg=%v", name, msg)) + }, + } +} + // AddChannel registers a channel to nodeInfo. // NOTE: NodeInfo must be of type DefaultNodeInfo else channels won't be updated // This is a bit messy at the moment but is cleaned up in the following version From 7bcf6d690363031596b32e2eb33aeb5977bf6040 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 21 Oct 2024 14:08:28 +0400 Subject: [PATCH 34/55] chore: not print congestion status --- p2p/transport.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/transport.go b/p2p/transport.go index cfcb5e8dac..4721ea12d6 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -389,7 +389,7 @@ func GetNewTracer() *logging.ConnectionTracer { //fmt.Println(fmt.Sprintf("UpdatedMTU: mtu=%v, done=%v", mtu, done)) }, UpdatedCongestionState: func(state logging.CongestionState) { - fmt.Println(fmt.Sprintf("UpdatedCongestionState: state=%v", state)) + //fmt.Println(fmt.Sprintf("UpdatedCongestionState: state=%v", state)) }, UpdatedPTOCount: func(value uint32) { //fmt.Println(fmt.Sprintf("UpdatedPTOCount: value=%v", value)) From abe9bb0952e82f352ac1e8106deec2b3de2bf3cc Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 21 Oct 2024 17:04:16 +0400 Subject: [PATCH 35/55] chore: remove unnecessaary todos --- p2p/peer.go | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index 2fc659ed39..49a7bcf377 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -319,17 +319,18 @@ func (p *peer) FlushStop() { p.metricsTicker.Stop() p.BaseService.OnStop() for _, stream := range p.streams { - // TODO(rach-id): set valid error codes stream.CancelRead(quic.StreamErrorCode(quic.NoError)) // stop everything and close the conn stream.CancelWrite(quic.StreamErrorCode(quic.NoError)) // stop everything and close the conn } + if err := p.conn.CloseWithError(quic.ApplicationErrorCode(quic.NoError), "stopping peer connection onStop"); err != nil { // stop everything and close the conn + p.Logger.Debug("Error while stopping peer", "err", err) + } } // OnStop implements BaseService. func (p *peer) OnStop() { p.metricsTicker.Stop() p.BaseService.OnStop() - // TODO(rach-id): set valid error code if err := p.conn.CloseWithError(quic.ApplicationErrorCode(quic.NoError), "stopping peer connection onStop"); err != nil { // stop everything and close the conn p.Logger.Debug("Error while stopping peer", "err", err) } @@ -491,7 +492,6 @@ func (p *peer) hasChannel(chID byte) bool { // CloseConn closes original connection. Used for cleaning up in cases where the peer had not been started at all. func (p *peer) CloseConn() error { - // TODO(rach-id): valid error code return p.conn.CloseWithError(quic.ApplicationErrorCode(quic.NoError), "closed peer connection closeCon") } @@ -509,7 +509,6 @@ func (p *peer) GetRemovalFailed() bool { // CloseConn closes the underlying connection func (pc *peerConn) CloseConn() { - // TODO(rach-id): valid error code pc.conn.CloseWithError(quic.ApplicationErrorCode(quic.NoError), "closed peer connection closeCon") } @@ -579,10 +578,6 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { } } - //if err := binary.Write(stream, binary.BigEndian, uint32(len(msgBytes))); err != nil { - // p.Logger.Error("Send len failed", "err", err, "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) - // return false - //} packet := p2p.Packet{ Sum: &p2p.Packet_PacketMsg{ PacketMsg: &p2p.PacketMsg{ @@ -593,13 +588,10 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { }, } _, err := protoio.NewDelimitedWriter(stream).WriteMsg(&packet) - //err := binary.Write(stream, binary.BigEndian, msgBytes) - p.Logger.Debug("sent data_len", "len", len(msgBytes)) if err != nil { - p.Logger.Error("Send failed", "channel", "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) + p.Logger.Debug("Send failed", "channel", "stream_id", stream.StreamID(), "msgBytes", log.NewLazySprintf("%X", msgBytes)) return false } - p.Logger.Debug("sent data", "channel", chID) labels := []string{ "peer_id", string(p.ID()), "chID", fmt.Sprintf("%#x", chID), @@ -613,13 +605,13 @@ func (p *peer) StartReceiving() error { for { stream, err := p.conn.AcceptStream(context.Background()) if err != nil { - p.Logger.Error("failed to accept stream", "err", err.Error()) + p.Logger.Debug("failed to accept stream", "err", err.Error()) return err } var chID byte err = binary.Read(stream, binary.BigEndian, &chID) if err != nil { - p.Logger.Error("failed to read channel ID", "err", err.Error()) + p.Logger.Debug("failed to read channel ID", "err", err.Error()) return err } // start accepting data @@ -628,12 +620,11 @@ func (p *peer) StartReceiving() error { var packet p2p.Packet _, err := protoio.NewDelimitedReader(stream, math.MaxInt32).ReadMsg(&packet) if err != nil { - p.Logger.Error("failed to read data from stream", "err", err.Error()) + p.Logger.Debug("failed to read data from stream", "err", err.Error()) return } dd := packet.Sum.(*p2p.Packet_PacketMsg) - p.Logger.Debug("received data", "channel", chID) p.onReceive(chID, dd.PacketMsg.Data) } }() From 68f7321961ac92d2e3de3dfb69c1c104ddd640f7 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 21 Oct 2024 17:13:58 +0400 Subject: [PATCH 36/55] chore: cleanup --- p2p/switch.go | 21 +-------- p2p/transport.go | 111 ++++++++++++----------------------------------- 2 files changed, 29 insertions(+), 103 deletions(-) diff --git a/p2p/switch.go b/p2p/switch.go index 48b74dae00..2911faacb4 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -372,10 +372,9 @@ func (sw *Switch) Peers() IPeerSet { // TODO: make record depending on reason. func (sw *Switch) StopPeerForError(peer Peer, reason interface{}) { if !peer.IsRunning() { - fmt.Println("Peer is not running!!!! we should stop but we CAN'T!!!!") + return } - sw.Logger.Error("Stopping peer for error", "peer", peer, "err", reason) sw.stopAndRemovePeer(peer, reason) if peer.IsPersistent() { @@ -422,14 +421,11 @@ func (sw *Switch) StopPeerGracefully(peer Peer) { func (sw *Switch) stopAndRemovePeer(peer Peer, reason interface{}) { sw.transport.Cleanup(peer) schema.WritePeerUpdate(sw.traceClient, string(peer.ID()), schema.PeerDisconnect, fmt.Sprintf("%v", reason)) - sw.Logger.Error("entering stop and remove peer", "peer", peer.ID()) if err := peer.Stop(); err != nil { sw.Logger.Error("error while stopping peer", "error", err) // TODO: should return error to be handled accordingly } - sw.Logger.Error("after peer stopped", "peer", peer.ID()) for _, reactor := range sw.reactors { - sw.Logger.Error("stopping peer reactors", "peer", peer.ID(), "reactor", reactor) reactor.RemovePeer(peer, reason) } @@ -437,7 +433,6 @@ func (sw *Switch) stopAndRemovePeer(peer Peer, reason interface{}) { // reconnect to our node and the switch calls InitPeer before // RemovePeer is finished. // https://github.com/tendermint/tendermint/issues/3338 - sw.Logger.Error("after removing all reactors", "peer", peer.ID()) if sw.peers.Remove(peer) { sw.metrics.Peers.Add(float64(-1)) } else { @@ -445,7 +440,6 @@ func (sw *Switch) stopAndRemovePeer(peer Peer, reason interface{}) { // We keep this message here as information to the developer. sw.Logger.Debug("error on peer removal", ",", "peer", peer.ID()) } - sw.Logger.Error("after removing peer from peers list", "peer", peer.ID()) } // reconnectToPeer tries to reconnect to the addr, first repeatedly @@ -922,20 +916,9 @@ func (sw *Switch) addPeer(p Peer) error { sw.Logger.Debug("Added peer", "peer", p) go func() { - fmt.Println("listening for connection to stop") - fmt.Println(p.ID()) - fmt.Println(p.RemoteAddr().String()) - fmt.Println("------------------------------------") select { case <-p.GetConnectionContext().Done(): - fmt.Println("----------------------------- connection stopped, context cancelled") - fmt.Println(p.ID()) - fmt.Println(p.RemoteAddr().String()) - fmt.Println("------------------------------------") - err := p.Stop() - if err != nil { - fmt.Println(err) - } + sw.Logger.Debug("counterparty connection closed", "peer_id", p.ID(), "remote_address", p.RemoteAddr().String()) sw.StopPeerForError(p, p.GetConnectionContext().Err()) } }() diff --git a/p2p/transport.go b/p2p/transport.go index 4721ea12d6..cb77f9c07b 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -10,6 +10,7 @@ import ( "github.com/quic-go/quic-go/logging" "github.com/tendermint/tendermint/libs/protoio" tmp2p "github.com/tendermint/tendermint/proto/tendermint/p2p" + "github.com/tendermint/tendermint/rpc/core" "net" "os" "runtime/debug" @@ -333,99 +334,44 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { } func GetNewTracer() *logging.ConnectionTracer { + logger := core.GetEnvironment().Logger return &logging.ConnectionTracer{ StartedConnection: func(local, remote net.Addr, srcConnID, destConnID quic.ConnectionID) { - fmt.Println(fmt.Sprintf("StartedConnection: local=%v, remote=%v, srcConnID=%v, destConnID=%v", local, remote, srcConnID, destConnID)) - }, - NegotiatedVersion: func(chosen quic.Version, clientVersions, serverVersions []quic.Version) { - //fmt.Println(fmt.Sprintf("NegotiatedVersion: chosen=%v, clientVersions=%v, serverVersions=%v", chosen, clientVersions, serverVersions)) + logger.Debug("StartedConnection", + "local", local, + "remote", remote, + "srcConnID", srcConnID, + "destConnID", destConnID, + ) }, ClosedConnection: func(err error) { - fmt.Println(fmt.Sprintf("ClosedConnection: error=%v", err)) - }, - SentTransportParameters: func(params *logging.TransportParameters) { - //fmt.Println(fmt.Sprintf("SentTransportParameters: params=%v", params)) - }, - ReceivedTransportParameters: func(params *logging.TransportParameters) { - //fmt.Println(fmt.Sprintf("ReceivedTransportParameters: params=%v", params)) - }, - RestoredTransportParameters: func(params *logging.TransportParameters) { - //fmt.Println(fmt.Sprintf("RestoredTransportParameters: params=%v", params)) - }, - SentLongHeaderPacket: func(header *logging.ExtendedHeader, byteCount logging.ByteCount, ecn logging.ECN, ack *logging.AckFrame, frames []logging.Frame) { - //fmt.Println(fmt.Sprintf("SentLongHeaderPacket: header=%v, byteCount=%v, ecn=%v, ack=%v, frames=%v", header, byteCount, ecn, ack, frames)) - }, - SentShortHeaderPacket: func(header *logging.ShortHeader, byteCount logging.ByteCount, ecn logging.ECN, ack *logging.AckFrame, frames []logging.Frame) { - //fmt.Println(fmt.Sprintf("SentShortHeaderPacket: header=%v, byteCount=%v, ecn=%v, ack=%v, frames=%v", header, byteCount, ecn, ack, frames)) - }, - ReceivedVersionNegotiationPacket: func(dest, src logging.ArbitraryLenConnectionID, versions []logging.Version) { - //fmt.Println(fmt.Sprintf("ReceivedVersionNegotiationPacket: dest=%v, src=%v, versions=%v", dest, src, versions)) - }, - ReceivedRetry: func(header *logging.Header) { - fmt.Println(fmt.Sprintf("ReceivedRetry: header=%v", header)) - }, - ReceivedLongHeaderPacket: func(header *logging.ExtendedHeader, byteCount logging.ByteCount, ecn logging.ECN, frames []logging.Frame) { - //fmt.Println(fmt.Sprintf("ReceivedLongHeaderPacket: header=%v, byteCount=%v, ecn=%v, frames=%v", header, byteCount, ecn, frames)) - }, - ReceivedShortHeaderPacket: func(header *logging.ShortHeader, byteCount logging.ByteCount, ecn logging.ECN, frames []logging.Frame) { - //fmt.Println(fmt.Sprintf("ReceivedShortHeaderPacket: header=%v, byteCount=%v, ecn=%v, frames=%v", header, byteCount, ecn, frames)) - }, - BufferedPacket: func(packetType logging.PacketType, byteCount logging.ByteCount) { - //fmt.Println(fmt.Sprintf("BufferedPacket: packetType=%v, byteCount=%v", packetType, byteCount)) + logger.Debug("ClosedConnection", + "error", err, + ) }, DroppedPacket: func(packetType logging.PacketType, packetNum logging.PacketNumber, byteCount logging.ByteCount, reason logging.PacketDropReason) { - fmt.Println(fmt.Sprintf("DroppedPacket: packetType=%v, packetNum=%v, byteCount=%v, reason=%v", packetType, packetNum, byteCount, reason)) - }, - UpdatedMetrics: func(rttStats *logging.RTTStats, cwnd, bytesInFlight logging.ByteCount, packetsInFlight int) { - //fmt.Println(fmt.Sprintf("UpdatedMetrics: rttStats=%v, cwnd=%v, bytesInFlight=%v, packetsInFlight=%v", rttStats, cwnd, bytesInFlight, packetsInFlight)) - }, - AcknowledgedPacket: func(encLevel logging.EncryptionLevel, packetNum logging.PacketNumber) { - //fmt.Println(fmt.Sprintf("AcknowledgedPacket: encLevel=%v, packetNum=%v", encLevel, packetNum)) + logger.Debug("DroppedPacket", + "packetType", packetType, + "packetNum", packetNum, + "byteCount", byteCount, + "reason", reason, + ) }, LostPacket: func(encLevel logging.EncryptionLevel, packetNum logging.PacketNumber, reason logging.PacketLossReason) { - fmt.Println(fmt.Sprintf("LostPacket: encLevel=%v, packetNum=%v, reason=%v", encLevel, packetNum, reason)) - }, - UpdatedMTU: func(mtu logging.ByteCount, done bool) { - //fmt.Println(fmt.Sprintf("UpdatedMTU: mtu=%v, done=%v", mtu, done)) - }, - UpdatedCongestionState: func(state logging.CongestionState) { - //fmt.Println(fmt.Sprintf("UpdatedCongestionState: state=%v", state)) - }, - UpdatedPTOCount: func(value uint32) { - //fmt.Println(fmt.Sprintf("UpdatedPTOCount: value=%v", value)) - }, - UpdatedKeyFromTLS: func(encLevel logging.EncryptionLevel, perspective logging.Perspective) { - //fmt.Println(fmt.Sprintf("UpdatedKeyFromTLS: encLevel=%v, perspective=%v", encLevel, perspective)) - }, - UpdatedKey: func(phase logging.KeyPhase, remote bool) { - //fmt.Println(fmt.Sprintf("UpdatedKey: phase=%v, remote=%v", phase, remote)) - }, - DroppedEncryptionLevel: func(encLevel logging.EncryptionLevel) { - //fmt.Println(fmt.Sprintf("DroppedEncryptionLevel: encLevel=%v", encLevel)) - }, - DroppedKey: func(phase logging.KeyPhase) { - //fmt.Println(fmt.Sprintf("DroppedKey: phase=%v", phase)) - }, - SetLossTimer: func(timerType logging.TimerType, encLevel logging.EncryptionLevel, t time.Time) { - //fmt.Println(fmt.Sprintf("SetLossTimer: timerType=%v, encLevel=%v, time=%v", timerType, encLevel, t)) - }, - LossTimerExpired: func(timerType logging.TimerType, encLevel logging.EncryptionLevel) { - //fmt.Println(fmt.Sprintf("LossTimerExpired: timerType=%v, encLevel=%v", timerType, encLevel)) - }, - LossTimerCanceled: func() { - //fmt.Println(fmt.Sprintf("LossTimerCanceled")) - }, - ECNStateUpdated: func(state logging.ECNState, trigger logging.ECNStateTrigger) { - //fmt.Println(fmt.Sprintf("ECNStateUpdated: state=%v, trigger=%v", state, trigger)) - }, - ChoseALPN: func(protocol string) { - //fmt.Println(fmt.Sprintf("ChoseALPN: protocol=%v", protocol)) + logger.Debug("LostPacket", + "encLevel", encLevel, + "packetNum", packetNum, + "reason", reason, + ) }, Close: func() { - fmt.Println(fmt.Sprintf("Close")) + logger.Debug("ClosedConnection") }, Debug: func(name, msg string) { - fmt.Println(fmt.Sprintf("Debug: name=%v, msg=%v", name, msg)) + logger.Debug("Debug", + "name", name, + "msg", msg, + ) }, } } @@ -478,7 +424,6 @@ func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { case mt.acceptc <- accept{err: err}: case <-mt.closec: // Give up if the transport was closed. - // TODO(rach-id): valid error code _ = c.CloseWithError(quic.ApplicationErrorCode(http3.ErrCodeConnectError), err.Error()) return } @@ -495,7 +440,6 @@ func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { _, nodeInfo, err = mt.getNodeInfo(c) if err == nil { addr := c.RemoteAddr() - //id := PubKeyToID(mt.nodeKey.PubKey()) netAddr = NewUDPNetAddress(nodeInfo.ID(), addr) } } @@ -529,7 +473,6 @@ func (mt *MultiplexTransport) cleanup(c quic.Connection) error { func (mt *MultiplexTransport) filterConn(c quic.Connection) (err error) { defer func() { if err != nil { - // TODO(rach-id): valid error _ = c.CloseWithError(quic.ApplicationErrorCode(http3.ErrCodeConnectError), err.Error()) } }() From 5ff281f9039c8ee58200f63f83f8c91786227b2a Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 21 Oct 2024 17:22:01 +0400 Subject: [PATCH 37/55] chore: use fmt instead of logger --- p2p/transport.go | 36 +++++++++--------------------------- 1 file changed, 9 insertions(+), 27 deletions(-) diff --git a/p2p/transport.go b/p2p/transport.go index cb77f9c07b..8e70127d17 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -10,7 +10,6 @@ import ( "github.com/quic-go/quic-go/logging" "github.com/tendermint/tendermint/libs/protoio" tmp2p "github.com/tendermint/tendermint/proto/tendermint/p2p" - "github.com/tendermint/tendermint/rpc/core" "net" "os" "runtime/debug" @@ -334,44 +333,27 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { } func GetNewTracer() *logging.ConnectionTracer { - logger := core.GetEnvironment().Logger return &logging.ConnectionTracer{ StartedConnection: func(local, remote net.Addr, srcConnID, destConnID quic.ConnectionID) { - logger.Debug("StartedConnection", - "local", local, - "remote", remote, - "srcConnID", srcConnID, - "destConnID", destConnID, - ) + fmt.Println(fmt.Sprintf("StartedConnection: local=%v, remote=%v, srcConnID=%v, destConnID=%v", local, remote, srcConnID, destConnID)) }, ClosedConnection: func(err error) { - logger.Debug("ClosedConnection", - "error", err, - ) + fmt.Println(fmt.Sprintf("ClosedConnection: error=%v", err)) + }, + ReceivedRetry: func(header *logging.Header) { + fmt.Println(fmt.Sprintf("ReceivedRetry: header=%v", header)) }, DroppedPacket: func(packetType logging.PacketType, packetNum logging.PacketNumber, byteCount logging.ByteCount, reason logging.PacketDropReason) { - logger.Debug("DroppedPacket", - "packetType", packetType, - "packetNum", packetNum, - "byteCount", byteCount, - "reason", reason, - ) + fmt.Println(fmt.Sprintf("DroppedPacket: packetType=%v, packetNum=%v, byteCount=%v, reason=%v", packetType, packetNum, byteCount, reason)) }, LostPacket: func(encLevel logging.EncryptionLevel, packetNum logging.PacketNumber, reason logging.PacketLossReason) { - logger.Debug("LostPacket", - "encLevel", encLevel, - "packetNum", packetNum, - "reason", reason, - ) + fmt.Println(fmt.Sprintf("LostPacket: encLevel=%v, packetNum=%v, reason=%v", encLevel, packetNum, reason)) }, Close: func() { - logger.Debug("ClosedConnection") + fmt.Println(fmt.Sprintf("ClosedConnection")) }, Debug: func(name, msg string) { - logger.Debug("Debug", - "name", name, - "msg", msg, - ) + fmt.Println(fmt.Sprintf("Debug: name=%v, msg=%v", name, msg)) }, } } From 257f9f896ed5aa750bd11fce4ea25aee61a40040 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Wed, 23 Oct 2024 12:36:00 +0400 Subject: [PATCH 38/55] chore: increase cert validity period --- p2p/certificate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/certificate.go b/p2p/certificate.go index 3c99110306..9fe4ff94ea 100644 --- a/p2p/certificate.go +++ b/p2p/certificate.go @@ -20,7 +20,7 @@ import ( // TODO(rach-id): mention this code is adapted from libp2p p2p/security/tls/crypto.go -const certValidityPeriod = 24 * time.Hour +const certValidityPeriod = 24 * 39 * time.Hour const certificatePrefix = "tendermint-tls:" // TODO(rach-id): update the OID prefix to reflect Celestia/Tendermint From 267a6c7c508f255148f25826b628ecd133439190 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Thu, 24 Oct 2024 00:53:16 +0400 Subject: [PATCH 39/55] chore: change the peer listen address to be the remote address --- p2p/transport.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/p2p/transport.go b/p2p/transport.go index 8e70127d17..379616b340 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -589,6 +589,11 @@ func exchangeNodeInfo( return nil, err } + fmt.Println("peer node info") + fmt.Println(peerNodeInfo) + + peerNodeInfo.ListenAddr = c.RemoteAddr().String() + return peerNodeInfo, nil } From 4da09a083651f577022ce17626a5002c8d59ec09 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Thu, 24 Oct 2024 01:36:18 +0400 Subject: [PATCH 40/55] chore: pex logs --- p2p/pex/addrbook.go | 30 +++++++++++++++++++++++++----- p2p/pex/pex_reactor.go | 20 ++++++++++++++++++++ p2p/switch.go | 2 ++ p2p/transport.go | 9 +++++++++ 4 files changed, 56 insertions(+), 5 deletions(-) diff --git a/p2p/pex/addrbook.go b/p2p/pex/addrbook.go index 63218692d3..03a09e9feb 100644 --- a/p2p/pex/addrbook.go +++ b/p2p/pex/addrbook.go @@ -393,6 +393,7 @@ func (a *addrBook) GetSelection() []*p2p.NetAddress { defer a.mtx.Unlock() bookSize := a.size() + fmt.Println(bookSize) if bookSize <= 0 { if bookSize < 0 { panic(fmt.Sprintf("Addrbook size %d (new: %d + old: %d) is less than 0", a.nNew+a.nOld, a.nNew, a.nOld)) @@ -405,14 +406,17 @@ func (a *addrBook) GetSelection() []*p2p.NetAddress { bookSize*getSelectionPercent/100) numAddresses = cmtmath.MinInt(maxGetSelection, numAddresses) + fmt.Println(numAddresses) // XXX: instead of making a list of all addresses, shuffling, and slicing a random chunk, // could we just select a random numAddresses of indexes? allAddr := make([]*p2p.NetAddress, bookSize) + fmt.Println(a.addrLookup) i := 0 for _, ka := range a.addrLookup { allAddr[i] = ka.Addr i++ } + fmt.Println(allAddr) // Fisher-Yates shuffle the array. We only need to do the first // `numAddresses' since we are throwing the rest. @@ -442,7 +446,10 @@ func (a *addrBook) GetSelectionWithBias(biasTowardsNewAddrs int) []*p2p.NetAddre a.mtx.Lock() defer a.mtx.Unlock() + fmt.Println(a.ourAddrs) bookSize := a.size() + fmt.Println("book size") + fmt.Println(bookSize) if bookSize <= 0 { if bookSize < 0 { panic(fmt.Sprintf("Addrbook size %d (new: %d + old: %d) is less than 0", a.nNew+a.nOld, a.nNew, a.nOld)) @@ -461,12 +468,16 @@ func (a *addrBook) GetSelectionWithBias(biasTowardsNewAddrs int) []*p2p.NetAddre cmtmath.MinInt(minGetSelection, bookSize), bookSize*getSelectionPercent/100) numAddresses = cmtmath.MinInt(maxGetSelection, numAddresses) + fmt.Println(numAddresses) // number of new addresses that, if possible, should be in the beginning of the selection // if there are no enough old addrs, will choose new addr instead. numRequiredNewAdd := cmtmath.MaxInt(percentageOfNum(biasTowardsNewAddrs, numAddresses), numAddresses-a.nOld) + fmt.Println(numRequiredNewAdd) selection := a.randomPickAddresses(bucketTypeNew, numRequiredNewAdd) + fmt.Println(selection) selection = append(selection, a.randomPickAddresses(bucketTypeOld, numAddresses-len(selection))...) + fmt.Println(selection) return selection } @@ -525,13 +536,16 @@ func (a *addrBook) getBucket(bucketType byte, bucketIdx int) map[string]*knownAd // NOTE: currently it always returns true. func (a *addrBook) addToNewBucket(ka *knownAddress, bucketIdx int) error { // Consistency check to ensure we don't add an already known address + fmt.Println("add to new bucket") if ka.isOld() { return errAddrBookOldAddressNewBucket{ka.Addr, bucketIdx} } + fmt.Println(1) addrStr := ka.Addr.String() bucket := a.getBucket(bucketTypeNew, bucketIdx) + fmt.Println(2) // Already exists? if _, ok := bucket[addrStr]; ok { return nil @@ -543,15 +557,18 @@ func (a *addrBook) addToNewBucket(ka *knownAddress, bucketIdx int) error { a.expireNew(bucketIdx) } + fmt.Println(3) // Add to bucket. bucket[addrStr] = ka // increment nNew if the peer doesnt already exist in a bucket if ka.addBucketRef(bucketIdx) == 1 { a.nNew++ } + fmt.Println(4) // Add it to addrLookup a.addrLookup[ka.ID()] = ka + fmt.Println(5) return nil } @@ -643,31 +660,33 @@ func (a *addrBook) addAddress(addr, src *p2p.NetAddress) error { return ErrAddrBookNilAddr{addr, src} } + fmt.Println(1) if err := addr.Valid(); err != nil { return ErrAddrBookInvalidAddr{Addr: addr, AddrErr: err} } + fmt.Println(2) if _, ok := a.badPeers[addr.ID]; ok { return ErrAddressBanned{addr} } - + fmt.Println(3) if _, ok := a.privateIDs[addr.ID]; ok { return ErrAddrBookPrivate{addr} } - + fmt.Println(4) if _, ok := a.privateIDs[src.ID]; ok { return ErrAddrBookPrivateSrc{src} } - + fmt.Println(5) // TODO: we should track ourAddrs by ID and by IP:PORT and refuse both. if _, ok := a.ourAddrs[addr.String()]; ok { return ErrAddrBookSelf{addr} } - + fmt.Println(6) if a.routabilityStrict && !addr.Routable() { return ErrAddrBookNonRoutable{addr} } - + fmt.Println(7) ka := a.addrLookup[addr.ID] if ka != nil { // If its already old and the address ID's are the same, ignore it. @@ -694,6 +713,7 @@ func (a *addrBook) addAddress(addr, src *p2p.NetAddress) error { if err != nil { return err } + fmt.Println(8) return a.addToNewBucket(ka, bucket) } diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 5fc24bfca2..34fda4ed8d 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -200,12 +200,14 @@ func (r *Reactor) AddPeer(p Peer) { r.RequestAddrs(p) } } else { + fmt.Println("---- .>>>> in add peeer ") // inbound peer is its own source addr, err := p.NodeInfo().NetAddress() if err != nil { r.Logger.Error("Failed to get peer NetAddress", "err", err, "peer", p) return } + fmt.Println(addr) // Make it explicit that addr and src are the same for an inbound peer. src := addr @@ -213,7 +215,11 @@ func (r *Reactor) AddPeer(p Peer) { // add to book. dont RequestAddrs right away because // we don't trust inbound as much - let ensurePeersRoutine handle it. err = r.book.AddAddress(addr, src) + fmt.Println(err) r.logErrAddrBook(err) + r.book.Save() + fmt.Println("....................................") + fmt.Println(r.book.GetSelection()) } } @@ -238,11 +244,14 @@ func (r *Reactor) logErrAddrBook(err error) { // Receive implements Reactor by handling incoming PEX messages. func (r *Reactor) ReceiveEnvelope(e p2p.Envelope) { + fmt.Println("receive enveloppe in pex") + fmt.Println(e) r.Logger.Debug("Received message", "src", e.Src, "chId", e.ChannelID, "msg", e.Message) switch msg := e.Message.(type) { case *tmp2p.PexRequest: + fmt.Println("pex request") // NOTE: this is a prime candidate for amplification attacks, // so it's important we // 1) restrict how frequently peers can request @@ -251,6 +260,8 @@ func (r *Reactor) ReceiveEnvelope(e p2p.Envelope) { // If we're a seed and this is an inbound peer, // respond once and disconnect. if r.config.SeedMode && !e.Src.IsOutbound() { + + fmt.Println("first close") id := string(e.Src.ID()) v := r.lastReceivedRequests.Get(id) if v != nil { @@ -269,6 +280,7 @@ func (r *Reactor) ReceiveEnvelope(e p2p.Envelope) { }() } else { + fmt.Println("else") // Check we're not receiving requests too frequently. if err := r.receiveRequest(e.Src); err != nil { r.Switch.StopPeerForError(e.Src, err) @@ -279,6 +291,7 @@ func (r *Reactor) ReceiveEnvelope(e p2p.Envelope) { } case *tmp2p.PexAddrs: + fmt.Println("pex addresses") // If we asked for addresses, add them to the book addrs, err := p2p.NetAddressesFromProto(msg.Addrs) if err != nil { @@ -310,6 +323,8 @@ func (r *Reactor) Receive(chID byte, peer p2p.Peer, msgBytes []byte) { if err != nil { panic(err) } + fmt.Println("in receive bo76dha") + fmt.Println(um) r.ReceiveEnvelope(p2p.Envelope{ ChannelID: chID, Src: peer, @@ -354,6 +369,7 @@ func (r *Reactor) receiveRequest(src Peer) error { // RequestAddrs asks peer for more addresses if we do not already have a // request out for this peer. func (r *Reactor) RequestAddrs(p Peer) { + fmt.Println("requesting addresses") id := string(p.ID()) if r.requestsSent.Has(id) { return @@ -370,6 +386,8 @@ func (r *Reactor) RequestAddrs(p Peer) { // request for this peer and deletes the open request. // If there's no open request for the src peer, it returns an error. func (r *Reactor) ReceiveAddrs(addrs []*p2p.NetAddress, src Peer) error { + fmt.Println("receved addresses") + fmt.Println(addrs) id := string(src.ID()) if !r.requestsSent.Has(id) { return ErrUnsolicitedList @@ -421,6 +439,8 @@ func (r *Reactor) ReceiveAddrs(addrs []*p2p.NetAddress, src Peer) error { // SendAddrs sends addrs to the peer. func (r *Reactor) SendAddrs(p Peer, netAddrs []*p2p.NetAddress) { + fmt.Println("sending addrs") + fmt.Println(netAddrs) e := p2p.Envelope{ ChannelID: PexChannel, Message: &tmp2p.PexAddrs{Addrs: p2p.NetAddressesToProto(netAddrs)}, diff --git a/p2p/switch.go b/p2p/switch.go index 2911faacb4..ae4ffaeaa9 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -908,6 +908,8 @@ func (sw *Switch) addPeer(p Peer) error { sw.metrics.Peers.Add(float64(1)) schema.WritePeerUpdate(sw.traceClient, string(p.ID()), schema.PeerJoin, "") + addr, _ := p.NodeInfo().NetAddress() + fmt.Println(addr) // Start all the reactor protocols on the peer. for _, reactor := range sw.reactors { reactor.AddPeer(p) diff --git a/p2p/transport.go b/p2p/transport.go index 379616b340..4160d4bdb3 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -426,6 +426,11 @@ func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { } } + fmt.Println("kelellelelelelellel") + fmt.Println(netAddr) + fmt.Println(nodeInfo) + fmt.Println("kelellelelelelellel") + select { case mt.acceptc <- accept{netAddr, c, nodeInfo, err}: // Make the upgraded peer available. @@ -510,6 +515,8 @@ func (mt *MultiplexTransport) getNodeInfo(c quic.Connection) (conn quic.Connecti isAuthFailure: true, } } + fmt.Println("remote node info") + fmt.Println(remoteNodeInfo) if err := remoteNodeInfo.Validate(); err != nil { return nil, nil, ErrRejected{ @@ -556,6 +563,8 @@ func exchangeNodeInfo( ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() + fmt.Println("our node info") + fmt.Println(ourNodeInfo) go func(errc chan<- error, c quic.Connection) { stream, err := c.OpenStreamSync(ctx) From f8b5204dd393974f09a3e1de76f87dee939b0665 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 28 Oct 2024 12:24:27 +0400 Subject: [PATCH 41/55] chore: ;use multiple channels to send --- p2p/peer.go | 121 +++++++++++++++++++++++++++++++++++++++++++---- p2p/transport.go | 2 - 2 files changed, 111 insertions(+), 12 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index 49a7bcf377..a0ee690e8b 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -185,7 +185,9 @@ type peer struct { nodeInfo NodeInfo channels []byte - streams map[byte]quic.Stream + streams map[byte]quic.Stream + blockPartStreams []quic.Stream + mempoolStreams []quic.Stream // User data Data *cmap.CMap @@ -217,15 +219,17 @@ func newPeer( options ...PeerOption, ) *peer { p := &peer{ - peerConn: pc, - nodeInfo: nodeInfo, - channels: nodeInfo.(DefaultNodeInfo).Channels, - Data: cmap.NewCMap(), - metricsTicker: time.NewTicker(metricsTickerDuration), - metrics: NopMetrics(), - mlc: mlc, - traceClient: trace.NoOpTracer(), - streams: make(map[byte]quic.Stream), + peerConn: pc, + nodeInfo: nodeInfo, + channels: nodeInfo.(DefaultNodeInfo).Channels, + Data: cmap.NewCMap(), + metricsTicker: time.NewTicker(metricsTickerDuration), + metrics: NopMetrics(), + mlc: mlc, + traceClient: trace.NoOpTracer(), + streams: make(map[byte]quic.Stream), + mempoolStreams: make([]quic.Stream, 0), + blockPartStreams: make([]quic.Stream, 0), } p.onReceive = func(chID byte, msgBytes []byte) { @@ -282,6 +286,14 @@ func newPeer( option(p) } + go func() { + err := p.initializeAboveStreams() + if err != nil { + p.Logger.Error("error initializing mempool and block part channels", "err", err.Error()) + onPeerError(p, err) + } + }() + return p } @@ -553,6 +565,35 @@ func (p *peer) metricsReporter() { //} } +func (p *peer) initializeAboveStreams() error { + p.Mutex.Lock() + defer p.Mutex.Unlock() + for i := 0; i < 10; i++ { + stream1, err := p.conn.OpenStreamSync(context.Background()) + if err != nil { + return err + } + err = binary.Write(stream1, binary.BigEndian, DataChannel) + if err != nil { + p.Logger.Error("error sending channel ID", "err", err.Error()) + return err + } + p.blockPartStreams = append(p.blockPartStreams, stream1) + + stream2, err := p.conn.OpenStreamSync(context.Background()) + if err != nil { + return err + } + err = binary.Write(stream2, binary.BigEndian, MempoolChannel) + if err != nil { + p.Logger.Error("error sending channel ID", "err", err.Error()) + return err + } + p.blockPartStreams = append(p.mempoolStreams, stream2) + } + return nil +} + // Send msg bytes to the channel identified by chID byte. Returns false if the // send queue is full after timeout, specified by MConnection. // SendEnvelope replaces Send which will be deprecated in a future release. @@ -562,6 +603,9 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { } else if !p.hasChannel(chID) { return false } + if chID == MempoolChannel || chID == DataChannel { + return p.sendOther(chID, msgBytes) + } stream, has := p.getStream(chID) if !has { newStream, err := p.conn.OpenStreamSync(context.Background()) @@ -601,6 +645,60 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { return true } +const MempoolChannel = byte(0x30) +const DataChannel = byte(0x21) + +func (p *peer) sendOther(id byte, bytes []byte) bool { + if len(bytes) == 0 { + return true + } + var send func([]byte) bool + if id == MempoolChannel { + send = func(bytes []byte) bool { + p.Mutex.Lock() + stream := p.mempoolStreams[bytes[len(bytes)/2]%10] + p.Mutex.Unlock() + packet := p2p.Packet{ + Sum: &p2p.Packet_PacketMsg{ + PacketMsg: &p2p.PacketMsg{ + ChannelID: int32(id), + EOF: true, + Data: bytes, + }, + }, + } + _, err := protoio.NewDelimitedWriter(stream).WriteMsg(&packet) + if err != nil { + p.Logger.Debug("Send failed", "channel", "stream_id", stream.StreamID(), "index", bytes[len(bytes)/2]%10, "msgBytes", log.NewLazySprintf("%X", bytes)) + return false + } + return true + } + } else { + send = func(bytes []byte) bool { + p.Mutex.Lock() + stream := p.blockPartStreams[bytes[len(bytes)/2]%10] + p.Mutex.Unlock() + packet := p2p.Packet{ + Sum: &p2p.Packet_PacketMsg{ + PacketMsg: &p2p.PacketMsg{ + ChannelID: int32(id), + EOF: true, + Data: bytes, + }, + }, + } + _, err := protoio.NewDelimitedWriter(stream).WriteMsg(&packet) + if err != nil { + p.Logger.Debug("Send failed", "channel", "stream_id", stream.StreamID(), "index", bytes[len(bytes)/2]%10, "msgBytes", log.NewLazySprintf("%X", bytes)) + return false + } + return true + } + } + return send(bytes) +} + func (p *peer) StartReceiving() error { for { stream, err := p.conn.AcceptStream(context.Background()) @@ -625,6 +723,9 @@ func (p *peer) StartReceiving() error { } dd := packet.Sum.(*p2p.Packet_PacketMsg) + if dd.PacketMsg.ChannelID != int32(chID) { + p.Logger.Error("received message on wrong channel", "expected channel id", chID, "received message channel id", dd.PacketMsg.ChannelID) + } p.onReceive(chID, dd.PacketMsg.Data) } }() diff --git a/p2p/transport.go b/p2p/transport.go index 4160d4bdb3..7bb9c6afef 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -601,8 +601,6 @@ func exchangeNodeInfo( fmt.Println("peer node info") fmt.Println(peerNodeInfo) - peerNodeInfo.ListenAddr = c.RemoteAddr().String() - return peerNodeInfo, nil } From 5570d4a9409c825e7ecf1c44845f8033b4e26ad8 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 28 Oct 2024 12:37:19 +0400 Subject: [PATCH 42/55] chore: ;use multiple channels to send --- p2p/peer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/peer.go b/p2p/peer.go index a0ee690e8b..3ec954119b 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -589,7 +589,7 @@ func (p *peer) initializeAboveStreams() error { p.Logger.Error("error sending channel ID", "err", err.Error()) return err } - p.blockPartStreams = append(p.mempoolStreams, stream2) + p.mempoolStreams = append(p.mempoolStreams, stream2) } return nil } From 71eadd18ea5fe6f6bdde782bcab6732a71c7449d Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 28 Oct 2024 13:10:43 +0400 Subject: [PATCH 43/55] chore: unnecessary fmt --- p2p/pex/addrbook.go | 30 +++++------------------------- p2p/pex/pex_reactor.go | 20 -------------------- p2p/transport.go | 13 ++++--------- 3 files changed, 9 insertions(+), 54 deletions(-) diff --git a/p2p/pex/addrbook.go b/p2p/pex/addrbook.go index 03a09e9feb..63218692d3 100644 --- a/p2p/pex/addrbook.go +++ b/p2p/pex/addrbook.go @@ -393,7 +393,6 @@ func (a *addrBook) GetSelection() []*p2p.NetAddress { defer a.mtx.Unlock() bookSize := a.size() - fmt.Println(bookSize) if bookSize <= 0 { if bookSize < 0 { panic(fmt.Sprintf("Addrbook size %d (new: %d + old: %d) is less than 0", a.nNew+a.nOld, a.nNew, a.nOld)) @@ -406,17 +405,14 @@ func (a *addrBook) GetSelection() []*p2p.NetAddress { bookSize*getSelectionPercent/100) numAddresses = cmtmath.MinInt(maxGetSelection, numAddresses) - fmt.Println(numAddresses) // XXX: instead of making a list of all addresses, shuffling, and slicing a random chunk, // could we just select a random numAddresses of indexes? allAddr := make([]*p2p.NetAddress, bookSize) - fmt.Println(a.addrLookup) i := 0 for _, ka := range a.addrLookup { allAddr[i] = ka.Addr i++ } - fmt.Println(allAddr) // Fisher-Yates shuffle the array. We only need to do the first // `numAddresses' since we are throwing the rest. @@ -446,10 +442,7 @@ func (a *addrBook) GetSelectionWithBias(biasTowardsNewAddrs int) []*p2p.NetAddre a.mtx.Lock() defer a.mtx.Unlock() - fmt.Println(a.ourAddrs) bookSize := a.size() - fmt.Println("book size") - fmt.Println(bookSize) if bookSize <= 0 { if bookSize < 0 { panic(fmt.Sprintf("Addrbook size %d (new: %d + old: %d) is less than 0", a.nNew+a.nOld, a.nNew, a.nOld)) @@ -468,16 +461,12 @@ func (a *addrBook) GetSelectionWithBias(biasTowardsNewAddrs int) []*p2p.NetAddre cmtmath.MinInt(minGetSelection, bookSize), bookSize*getSelectionPercent/100) numAddresses = cmtmath.MinInt(maxGetSelection, numAddresses) - fmt.Println(numAddresses) // number of new addresses that, if possible, should be in the beginning of the selection // if there are no enough old addrs, will choose new addr instead. numRequiredNewAdd := cmtmath.MaxInt(percentageOfNum(biasTowardsNewAddrs, numAddresses), numAddresses-a.nOld) - fmt.Println(numRequiredNewAdd) selection := a.randomPickAddresses(bucketTypeNew, numRequiredNewAdd) - fmt.Println(selection) selection = append(selection, a.randomPickAddresses(bucketTypeOld, numAddresses-len(selection))...) - fmt.Println(selection) return selection } @@ -536,16 +525,13 @@ func (a *addrBook) getBucket(bucketType byte, bucketIdx int) map[string]*knownAd // NOTE: currently it always returns true. func (a *addrBook) addToNewBucket(ka *knownAddress, bucketIdx int) error { // Consistency check to ensure we don't add an already known address - fmt.Println("add to new bucket") if ka.isOld() { return errAddrBookOldAddressNewBucket{ka.Addr, bucketIdx} } - fmt.Println(1) addrStr := ka.Addr.String() bucket := a.getBucket(bucketTypeNew, bucketIdx) - fmt.Println(2) // Already exists? if _, ok := bucket[addrStr]; ok { return nil @@ -557,18 +543,15 @@ func (a *addrBook) addToNewBucket(ka *knownAddress, bucketIdx int) error { a.expireNew(bucketIdx) } - fmt.Println(3) // Add to bucket. bucket[addrStr] = ka // increment nNew if the peer doesnt already exist in a bucket if ka.addBucketRef(bucketIdx) == 1 { a.nNew++ } - fmt.Println(4) // Add it to addrLookup a.addrLookup[ka.ID()] = ka - fmt.Println(5) return nil } @@ -660,33 +643,31 @@ func (a *addrBook) addAddress(addr, src *p2p.NetAddress) error { return ErrAddrBookNilAddr{addr, src} } - fmt.Println(1) if err := addr.Valid(); err != nil { return ErrAddrBookInvalidAddr{Addr: addr, AddrErr: err} } - fmt.Println(2) if _, ok := a.badPeers[addr.ID]; ok { return ErrAddressBanned{addr} } - fmt.Println(3) + if _, ok := a.privateIDs[addr.ID]; ok { return ErrAddrBookPrivate{addr} } - fmt.Println(4) + if _, ok := a.privateIDs[src.ID]; ok { return ErrAddrBookPrivateSrc{src} } - fmt.Println(5) + // TODO: we should track ourAddrs by ID and by IP:PORT and refuse both. if _, ok := a.ourAddrs[addr.String()]; ok { return ErrAddrBookSelf{addr} } - fmt.Println(6) + if a.routabilityStrict && !addr.Routable() { return ErrAddrBookNonRoutable{addr} } - fmt.Println(7) + ka := a.addrLookup[addr.ID] if ka != nil { // If its already old and the address ID's are the same, ignore it. @@ -713,7 +694,6 @@ func (a *addrBook) addAddress(addr, src *p2p.NetAddress) error { if err != nil { return err } - fmt.Println(8) return a.addToNewBucket(ka, bucket) } diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 34fda4ed8d..5fc24bfca2 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -200,14 +200,12 @@ func (r *Reactor) AddPeer(p Peer) { r.RequestAddrs(p) } } else { - fmt.Println("---- .>>>> in add peeer ") // inbound peer is its own source addr, err := p.NodeInfo().NetAddress() if err != nil { r.Logger.Error("Failed to get peer NetAddress", "err", err, "peer", p) return } - fmt.Println(addr) // Make it explicit that addr and src are the same for an inbound peer. src := addr @@ -215,11 +213,7 @@ func (r *Reactor) AddPeer(p Peer) { // add to book. dont RequestAddrs right away because // we don't trust inbound as much - let ensurePeersRoutine handle it. err = r.book.AddAddress(addr, src) - fmt.Println(err) r.logErrAddrBook(err) - r.book.Save() - fmt.Println("....................................") - fmt.Println(r.book.GetSelection()) } } @@ -244,14 +238,11 @@ func (r *Reactor) logErrAddrBook(err error) { // Receive implements Reactor by handling incoming PEX messages. func (r *Reactor) ReceiveEnvelope(e p2p.Envelope) { - fmt.Println("receive enveloppe in pex") - fmt.Println(e) r.Logger.Debug("Received message", "src", e.Src, "chId", e.ChannelID, "msg", e.Message) switch msg := e.Message.(type) { case *tmp2p.PexRequest: - fmt.Println("pex request") // NOTE: this is a prime candidate for amplification attacks, // so it's important we // 1) restrict how frequently peers can request @@ -260,8 +251,6 @@ func (r *Reactor) ReceiveEnvelope(e p2p.Envelope) { // If we're a seed and this is an inbound peer, // respond once and disconnect. if r.config.SeedMode && !e.Src.IsOutbound() { - - fmt.Println("first close") id := string(e.Src.ID()) v := r.lastReceivedRequests.Get(id) if v != nil { @@ -280,7 +269,6 @@ func (r *Reactor) ReceiveEnvelope(e p2p.Envelope) { }() } else { - fmt.Println("else") // Check we're not receiving requests too frequently. if err := r.receiveRequest(e.Src); err != nil { r.Switch.StopPeerForError(e.Src, err) @@ -291,7 +279,6 @@ func (r *Reactor) ReceiveEnvelope(e p2p.Envelope) { } case *tmp2p.PexAddrs: - fmt.Println("pex addresses") // If we asked for addresses, add them to the book addrs, err := p2p.NetAddressesFromProto(msg.Addrs) if err != nil { @@ -323,8 +310,6 @@ func (r *Reactor) Receive(chID byte, peer p2p.Peer, msgBytes []byte) { if err != nil { panic(err) } - fmt.Println("in receive bo76dha") - fmt.Println(um) r.ReceiveEnvelope(p2p.Envelope{ ChannelID: chID, Src: peer, @@ -369,7 +354,6 @@ func (r *Reactor) receiveRequest(src Peer) error { // RequestAddrs asks peer for more addresses if we do not already have a // request out for this peer. func (r *Reactor) RequestAddrs(p Peer) { - fmt.Println("requesting addresses") id := string(p.ID()) if r.requestsSent.Has(id) { return @@ -386,8 +370,6 @@ func (r *Reactor) RequestAddrs(p Peer) { // request for this peer and deletes the open request. // If there's no open request for the src peer, it returns an error. func (r *Reactor) ReceiveAddrs(addrs []*p2p.NetAddress, src Peer) error { - fmt.Println("receved addresses") - fmt.Println(addrs) id := string(src.ID()) if !r.requestsSent.Has(id) { return ErrUnsolicitedList @@ -439,8 +421,6 @@ func (r *Reactor) ReceiveAddrs(addrs []*p2p.NetAddress, src Peer) error { // SendAddrs sends addrs to the peer. func (r *Reactor) SendAddrs(p Peer, netAddrs []*p2p.NetAddress) { - fmt.Println("sending addrs") - fmt.Println(netAddrs) e := p2p.Envelope{ ChannelID: PexChannel, Message: &tmp2p.PexAddrs{Addrs: p2p.NetAddressesToProto(netAddrs)}, diff --git a/p2p/transport.go b/p2p/transport.go index 7bb9c6afef..d3e8c1dbcb 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -341,19 +341,19 @@ func GetNewTracer() *logging.ConnectionTracer { fmt.Println(fmt.Sprintf("ClosedConnection: error=%v", err)) }, ReceivedRetry: func(header *logging.Header) { - fmt.Println(fmt.Sprintf("ReceivedRetry: header=%v", header)) + //fmt.Println(fmt.Sprintf("ReceivedRetry: header=%v", header)) }, DroppedPacket: func(packetType logging.PacketType, packetNum logging.PacketNumber, byteCount logging.ByteCount, reason logging.PacketDropReason) { - fmt.Println(fmt.Sprintf("DroppedPacket: packetType=%v, packetNum=%v, byteCount=%v, reason=%v", packetType, packetNum, byteCount, reason)) + //fmt.Println(fmt.Sprintf("DroppedPacket: packetType=%v, packetNum=%v, byteCount=%v, reason=%v", packetType, packetNum, byteCount, reason)) }, LostPacket: func(encLevel logging.EncryptionLevel, packetNum logging.PacketNumber, reason logging.PacketLossReason) { - fmt.Println(fmt.Sprintf("LostPacket: encLevel=%v, packetNum=%v, reason=%v", encLevel, packetNum, reason)) + //fmt.Println(fmt.Sprintf("LostPacket: encLevel=%v, packetNum=%v, reason=%v", encLevel, packetNum, reason)) }, Close: func() { fmt.Println(fmt.Sprintf("ClosedConnection")) }, Debug: func(name, msg string) { - fmt.Println(fmt.Sprintf("Debug: name=%v, msg=%v", name, msg)) + //fmt.Println(fmt.Sprintf("Debug: name=%v, msg=%v", name, msg)) }, } } @@ -426,11 +426,6 @@ func (mt *MultiplexTransport) acceptPeers(ctx context.Context) { } } - fmt.Println("kelellelelelelellel") - fmt.Println(netAddr) - fmt.Println(nodeInfo) - fmt.Println("kelellelelelelellel") - select { case mt.acceptc <- accept{netAddr, c, nodeInfo, err}: // Make the upgraded peer available. From 0a060824ccaf56c163f8e1a1170df6988f856176 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 28 Oct 2024 13:35:10 +0400 Subject: [PATCH 44/55] chore: use rand to choose stream --- p2p/peer.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index 3ec954119b..5838d57a44 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "github.com/quic-go/quic-go" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/libs/protoio" "github.com/tendermint/tendermint/pkg/trace/schema" "github.com/tendermint/tendermint/proto/tendermint/p2p" @@ -656,7 +657,7 @@ func (p *peer) sendOther(id byte, bytes []byte) bool { if id == MempoolChannel { send = func(bytes []byte) bool { p.Mutex.Lock() - stream := p.mempoolStreams[bytes[len(bytes)/2]%10] + stream := p.mempoolStreams[int(crypto.CRandBytes(1)[0])%10] p.Mutex.Unlock() packet := p2p.Packet{ Sum: &p2p.Packet_PacketMsg{ @@ -677,7 +678,7 @@ func (p *peer) sendOther(id byte, bytes []byte) bool { } else { send = func(bytes []byte) bool { p.Mutex.Lock() - stream := p.blockPartStreams[bytes[len(bytes)/2]%10] + stream := p.blockPartStreams[int(crypto.CRandBytes(1)[0])%10] p.Mutex.Unlock() packet := p2p.Packet{ Sum: &p2p.Packet_PacketMsg{ From 46c8386103f07624f5b7ffffe7337060ff73ea6d Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 28 Oct 2024 13:36:11 +0400 Subject: [PATCH 45/55] chore: use rand to choose stream --- p2p/peer.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index 5838d57a44..9d3ac964fd 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -656,8 +656,10 @@ func (p *peer) sendOther(id byte, bytes []byte) bool { var send func([]byte) bool if id == MempoolChannel { send = func(bytes []byte) bool { + rnd := int(crypto.CRandBytes(1)[0]) % 10 + fmt.Println(rnd) p.Mutex.Lock() - stream := p.mempoolStreams[int(crypto.CRandBytes(1)[0])%10] + stream := p.mempoolStreams[rnd] p.Mutex.Unlock() packet := p2p.Packet{ Sum: &p2p.Packet_PacketMsg{ @@ -677,8 +679,10 @@ func (p *peer) sendOther(id byte, bytes []byte) bool { } } else { send = func(bytes []byte) bool { + rnd := int(crypto.CRandBytes(1)[0]) % 10 + fmt.Println(rnd) p.Mutex.Lock() - stream := p.blockPartStreams[int(crypto.CRandBytes(1)[0])%10] + stream := p.blockPartStreams[rnd] p.Mutex.Unlock() packet := p2p.Packet{ Sum: &p2p.Packet_PacketMsg{ From dd2de41d687462b7ad801b921ac5cf8b7f4d8bad Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 28 Oct 2024 14:00:59 +0400 Subject: [PATCH 46/55] chore: rmove fmt --- p2p/peer.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index 9d3ac964fd..92c5ccd3b0 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -657,7 +657,6 @@ func (p *peer) sendOther(id byte, bytes []byte) bool { if id == MempoolChannel { send = func(bytes []byte) bool { rnd := int(crypto.CRandBytes(1)[0]) % 10 - fmt.Println(rnd) p.Mutex.Lock() stream := p.mempoolStreams[rnd] p.Mutex.Unlock() @@ -680,7 +679,6 @@ func (p *peer) sendOther(id byte, bytes []byte) bool { } else { send = func(bytes []byte) bool { rnd := int(crypto.CRandBytes(1)[0]) % 10 - fmt.Println(rnd) p.Mutex.Lock() stream := p.blockPartStreams[rnd] p.Mutex.Unlock() From 9d8adf64d576def525e8812a4afbb2cc7df9fca4 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 28 Oct 2024 14:17:27 +0400 Subject: [PATCH 47/55] chore: increase number of streams to 50 --- p2p/peer.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index 92c5ccd3b0..f28075c904 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -566,10 +566,12 @@ func (p *peer) metricsReporter() { //} } +const totalStream = 50 + func (p *peer) initializeAboveStreams() error { p.Mutex.Lock() defer p.Mutex.Unlock() - for i := 0; i < 10; i++ { + for i := 0; i < totalStream; i++ { stream1, err := p.conn.OpenStreamSync(context.Background()) if err != nil { return err @@ -656,7 +658,7 @@ func (p *peer) sendOther(id byte, bytes []byte) bool { var send func([]byte) bool if id == MempoolChannel { send = func(bytes []byte) bool { - rnd := int(crypto.CRandBytes(1)[0]) % 10 + rnd := int(crypto.CRandBytes(1)[0]) % totalStream p.Mutex.Lock() stream := p.mempoolStreams[rnd] p.Mutex.Unlock() @@ -678,7 +680,7 @@ func (p *peer) sendOther(id byte, bytes []byte) bool { } } else { send = func(bytes []byte) bool { - rnd := int(crypto.CRandBytes(1)[0]) % 10 + rnd := int(crypto.CRandBytes(1)[0]) % totalStream p.Mutex.Lock() stream := p.blockPartStreams[rnd] p.Mutex.Unlock() From 5e234c0064d4fdb26ca61fc67d848ab06e198efb Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 28 Oct 2024 14:39:10 +0400 Subject: [PATCH 48/55] chore: create streams for other reactors too --- p2p/peer.go | 127 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 119 insertions(+), 8 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index f28075c904..62f4431015 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -186,9 +186,12 @@ type peer struct { nodeInfo NodeInfo channels []byte - streams map[byte]quic.Stream - blockPartStreams []quic.Stream - mempoolStreams []quic.Stream + streams map[byte]quic.Stream + blockPartStreams []quic.Stream + mempoolStreams []quic.Stream + blockchainStreams []quic.Stream + snapshotStreams []quic.Stream + chunkStreams []quic.Stream // User data Data *cmap.CMap @@ -566,7 +569,7 @@ func (p *peer) metricsReporter() { //} } -const totalStream = 50 +const totalStream = 40 func (p *peer) initializeAboveStreams() error { p.Mutex.Lock() @@ -593,6 +596,39 @@ func (p *peer) initializeAboveStreams() error { return err } p.mempoolStreams = append(p.mempoolStreams, stream2) + + stream3, err := p.conn.OpenStreamSync(context.Background()) + if err != nil { + return err + } + err = binary.Write(stream3, binary.BigEndian, BlockchainChannel) + if err != nil { + p.Logger.Error("error sending channel ID", "err", err.Error()) + return err + } + p.blockchainStreams = append(p.blockchainStreams, stream3) + + stream4, err := p.conn.OpenStreamSync(context.Background()) + if err != nil { + return err + } + err = binary.Write(stream4, binary.BigEndian, SnapshotChannel) + if err != nil { + p.Logger.Error("error sending channel ID", "err", err.Error()) + return err + } + p.snapshotStreams = append(p.snapshotStreams, stream4) + + stream5, err := p.conn.OpenStreamSync(context.Background()) + if err != nil { + return err + } + err = binary.Write(stream5, binary.BigEndian, ChunkChannel) + if err != nil { + p.Logger.Error("error sending channel ID", "err", err.Error()) + return err + } + p.chunkStreams = append(p.chunkStreams, stream5) } return nil } @@ -606,7 +642,11 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { } else if !p.hasChannel(chID) { return false } - if chID == MempoolChannel || chID == DataChannel { + if chID == MempoolChannel || + chID == DataChannel || + chID == BlockchainChannel || + chID == SnapshotChannel || + chID == ChunkChannel { return p.sendOther(chID, msgBytes) } stream, has := p.getStream(chID) @@ -648,15 +688,86 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { return true } -const MempoolChannel = byte(0x30) -const DataChannel = byte(0x21) +const ( + BlockchainChannel = byte(0x40) + SnapshotChannel = byte(0x60) + ChunkChannel = byte(0x61) + MempoolChannel = byte(0x30) + DataChannel = byte(0x21) +) func (p *peer) sendOther(id byte, bytes []byte) bool { if len(bytes) == 0 { return true } var send func([]byte) bool - if id == MempoolChannel { + if id == BlockchainChannel { + send = func(bytes []byte) bool { + rnd := int(crypto.CRandBytes(1)[0]) % totalStream + p.Mutex.Lock() + stream := p.blockchainStreams[rnd] + p.Mutex.Unlock() + packet := p2p.Packet{ + Sum: &p2p.Packet_PacketMsg{ + PacketMsg: &p2p.PacketMsg{ + ChannelID: int32(id), + EOF: true, + Data: bytes, + }, + }, + } + _, err := protoio.NewDelimitedWriter(stream).WriteMsg(&packet) + if err != nil { + p.Logger.Debug("Send failed", "channel", "stream_id", stream.StreamID(), "index", bytes[len(bytes)/2]%10, "msgBytes", log.NewLazySprintf("%X", bytes)) + return false + } + return true + } + } else if id == SnapshotChannel { + send = func(bytes []byte) bool { + rnd := int(crypto.CRandBytes(1)[0]) % totalStream + p.Mutex.Lock() + stream := p.snapshotStreams[rnd] + p.Mutex.Unlock() + packet := p2p.Packet{ + Sum: &p2p.Packet_PacketMsg{ + PacketMsg: &p2p.PacketMsg{ + ChannelID: int32(id), + EOF: true, + Data: bytes, + }, + }, + } + _, err := protoio.NewDelimitedWriter(stream).WriteMsg(&packet) + if err != nil { + p.Logger.Debug("Send failed", "channel", "stream_id", stream.StreamID(), "index", bytes[len(bytes)/2]%10, "msgBytes", log.NewLazySprintf("%X", bytes)) + return false + } + return true + } + } else if id == ChunkChannel { + send = func(bytes []byte) bool { + rnd := int(crypto.CRandBytes(1)[0]) % totalStream + p.Mutex.Lock() + stream := p.chunkStreams[rnd] + p.Mutex.Unlock() + packet := p2p.Packet{ + Sum: &p2p.Packet_PacketMsg{ + PacketMsg: &p2p.PacketMsg{ + ChannelID: int32(id), + EOF: true, + Data: bytes, + }, + }, + } + _, err := protoio.NewDelimitedWriter(stream).WriteMsg(&packet) + if err != nil { + p.Logger.Debug("Send failed", "channel", "stream_id", stream.StreamID(), "index", bytes[len(bytes)/2]%10, "msgBytes", log.NewLazySprintf("%X", bytes)) + return false + } + return true + } + } else if id == MempoolChannel { send = func(bytes []byte) bool { rnd := int(crypto.CRandBytes(1)[0]) % totalStream p.Mutex.Lock() From c53eff209c9ed3fb8e51ce79400e2b2347736ab9 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 28 Oct 2024 14:43:30 +0400 Subject: [PATCH 49/55] chore: inmitlaize correctly satreams --- p2p/peer.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index 62f4431015..43f48ef6a5 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -223,17 +223,20 @@ func newPeer( options ...PeerOption, ) *peer { p := &peer{ - peerConn: pc, - nodeInfo: nodeInfo, - channels: nodeInfo.(DefaultNodeInfo).Channels, - Data: cmap.NewCMap(), - metricsTicker: time.NewTicker(metricsTickerDuration), - metrics: NopMetrics(), - mlc: mlc, - traceClient: trace.NoOpTracer(), - streams: make(map[byte]quic.Stream), - mempoolStreams: make([]quic.Stream, 0), - blockPartStreams: make([]quic.Stream, 0), + peerConn: pc, + nodeInfo: nodeInfo, + channels: nodeInfo.(DefaultNodeInfo).Channels, + Data: cmap.NewCMap(), + metricsTicker: time.NewTicker(metricsTickerDuration), + metrics: NopMetrics(), + mlc: mlc, + traceClient: trace.NoOpTracer(), + streams: make(map[byte]quic.Stream), + mempoolStreams: make([]quic.Stream, 0), + blockPartStreams: make([]quic.Stream, 0), + blockchainStreams: make([]quic.Stream, 0), + snapshotStreams: make([]quic.Stream, 0), + chunkStreams: make([]quic.Stream, 0), } p.onReceive = func(chID byte, msgBytes []byte) { From b25b789fd4dd75f9e5ec962ee50a7aaf04b52409 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Mon, 28 Oct 2024 14:50:56 +0400 Subject: [PATCH 50/55] chore: create streams sync --- p2p/peer.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index 43f48ef6a5..722ffb70b6 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -293,13 +293,10 @@ func newPeer( option(p) } - go func() { - err := p.initializeAboveStreams() - if err != nil { - p.Logger.Error("error initializing mempool and block part channels", "err", err.Error()) - onPeerError(p, err) - } - }() + err := p.initializeAboveStreams() + if err != nil { + p.Logger.Error("error initializing mempool and block part channels", "err", err.Error()) + } return p } From 2fa96bc9156a9c8f4125590e7f72b74cb00c4417 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Tue, 29 Oct 2024 12:54:17 +0400 Subject: [PATCH 51/55] crazy: send in a go routine and return immediatly --- p2p/peer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/p2p/peer.go b/p2p/peer.go index 722ffb70b6..84254b02b6 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -87,7 +87,8 @@ func SendEnvelopeShim(p Peer, e Envelope, lg log.Logger) bool { lg.Error("marshaling message to send", "error", err) return false } - return p.Send(e.ChannelID, msgBytes) + go p.Send(e.ChannelID, msgBytes) + return true } // EnvelopeTrySendShim implements a shim to allow the legacy peer type that From 33d1db4a75409f34e344b0318d5b6229631688a0 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Tue, 29 Oct 2024 15:29:07 +0400 Subject: [PATCH 52/55] chore: revert crazy and uses deterministic way of choosing from multiple streams --- p2p/peer.go | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/p2p/peer.go b/p2p/peer.go index 84254b02b6..637b3906f1 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "fmt" "github.com/quic-go/quic-go" - "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/libs/protoio" "github.com/tendermint/tendermint/pkg/trace/schema" "github.com/tendermint/tendermint/proto/tendermint/p2p" @@ -87,7 +86,7 @@ func SendEnvelopeShim(p Peer, e Envelope, lg log.Logger) bool { lg.Error("marshaling message to send", "error", err) return false } - go p.Send(e.ChannelID, msgBytes) + p.Send(e.ChannelID, msgBytes) return true } @@ -697,6 +696,14 @@ const ( DataChannel = byte(0x21) ) +var ( + blockchainCount = int64(0) + snapshotCount = int64(0) + chunkCount = int64(0) + memCount = int64(0) + partsCount = int64(0) +) + func (p *peer) sendOther(id byte, bytes []byte) bool { if len(bytes) == 0 { return true @@ -704,9 +711,11 @@ func (p *peer) sendOther(id byte, bytes []byte) bool { var send func([]byte) bool if id == BlockchainChannel { send = func(bytes []byte) bool { - rnd := int(crypto.CRandBytes(1)[0]) % totalStream + defer func() { + blockchainCount++ + }() p.Mutex.Lock() - stream := p.blockchainStreams[rnd] + stream := p.blockchainStreams[blockchainCount%totalStream] p.Mutex.Unlock() packet := p2p.Packet{ Sum: &p2p.Packet_PacketMsg{ @@ -726,9 +735,11 @@ func (p *peer) sendOther(id byte, bytes []byte) bool { } } else if id == SnapshotChannel { send = func(bytes []byte) bool { - rnd := int(crypto.CRandBytes(1)[0]) % totalStream + defer func() { + snapshotCount++ + }() p.Mutex.Lock() - stream := p.snapshotStreams[rnd] + stream := p.snapshotStreams[snapshotCount%totalStream] p.Mutex.Unlock() packet := p2p.Packet{ Sum: &p2p.Packet_PacketMsg{ @@ -748,9 +759,11 @@ func (p *peer) sendOther(id byte, bytes []byte) bool { } } else if id == ChunkChannel { send = func(bytes []byte) bool { - rnd := int(crypto.CRandBytes(1)[0]) % totalStream + defer func() { + chunkCount++ + }() p.Mutex.Lock() - stream := p.chunkStreams[rnd] + stream := p.chunkStreams[chunkCount%totalStream] p.Mutex.Unlock() packet := p2p.Packet{ Sum: &p2p.Packet_PacketMsg{ @@ -770,9 +783,11 @@ func (p *peer) sendOther(id byte, bytes []byte) bool { } } else if id == MempoolChannel { send = func(bytes []byte) bool { - rnd := int(crypto.CRandBytes(1)[0]) % totalStream + defer func() { + memCount++ + }() p.Mutex.Lock() - stream := p.mempoolStreams[rnd] + stream := p.mempoolStreams[memCount%totalStream] p.Mutex.Unlock() packet := p2p.Packet{ Sum: &p2p.Packet_PacketMsg{ @@ -792,9 +807,11 @@ func (p *peer) sendOther(id byte, bytes []byte) bool { } } else { send = func(bytes []byte) bool { - rnd := int(crypto.CRandBytes(1)[0]) % totalStream + defer func() { + partsCount++ + }() p.Mutex.Lock() - stream := p.blockPartStreams[rnd] + stream := p.blockPartStreams[partsCount%totalStream] p.Mutex.Unlock() packet := p2p.Packet{ Sum: &p2p.Packet_PacketMsg{ From ffe88efdd15f9edc4a3e76962c1ba181f9de6117 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Wed, 30 Oct 2024 10:44:21 +0400 Subject: [PATCH 53/55] chore: increase max block size --- types/params.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/params.go b/types/params.go index 2fc704f115..d58906155e 100644 --- a/types/params.go +++ b/types/params.go @@ -12,7 +12,7 @@ import ( const ( // MaxBlockSizeBytes is the maximum permitted size of the blocks. - MaxBlockSizeBytes = 104857600 // 100MB + MaxBlockSizeBytes = 1048576000 // 100MB // BlockPartSizeBytes is the size of one block part. BlockPartSizeBytes uint32 = 65536 // 64kB From 7c3959131587821b3b4328090ebf8cdfc893d532 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Wed, 13 Nov 2024 14:45:23 +0400 Subject: [PATCH 54/55] chore(starting to degug): ignore peer has no state panic --- consensus/reactor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/reactor.go b/consensus/reactor.go index 37feec35ed..f2fb5cd111 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -262,7 +262,7 @@ func (conR *Reactor) ReceiveEnvelope(e p2p.Envelope) { // Get peer states ps, ok := e.Src.Get(types.PeerStateKey).(*PeerState) if !ok { - panic(fmt.Sprintf("Peer %v has no state", e.Src)) + return } switch e.ChannelID { From 7635b86b7875b99a47c95a7bcd803ccf6d239365 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Fri, 15 Nov 2024 13:29:21 +0400 Subject: [PATCH 55/55] chore: crank up some params --- p2p/netaddress.go | 14 +++++++++----- p2p/transport.go | 16 ++++++++++------ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/p2p/netaddress.go b/p2p/netaddress.go index c5c3bde6c2..c4bce27b2b 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -287,11 +287,15 @@ func (na *NetAddress) DialTimeout(timeout time.Duration, tlsConf *tls.Config) (q ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() quickConfig := quic.Config{ - MaxIdleTimeout: time.Minute, - MaxIncomingStreams: 10000, - MaxIncomingUniStreams: 10000, - KeepAlivePeriod: 10 * time.Second, - EnableDatagrams: true, + InitialStreamReceiveWindow: 10_000_000_000, + MaxStreamReceiveWindow: 10_000_000_000, + InitialConnectionReceiveWindow: 10_000_000_000, + MaxConnectionReceiveWindow: 10_000_000_000, + MaxIncomingStreams: 1000000000, + MaxIncomingUniStreams: 1000000000, + MaxIdleTimeout: time.Minute, + KeepAlivePeriod: 10 * time.Second, + EnableDatagrams: true, Tracer: func(ctx context.Context, perspective logging.Perspective, id quic.ConnectionID) *logging.ConnectionTracer { return logging.NewMultiplexedConnectionTracer(GetNewTracer()) }, diff --git a/p2p/transport.go b/p2p/transport.go index d3e8c1dbcb..a74a00868d 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -308,13 +308,17 @@ func (mt *MultiplexTransport) Listen(addr NetAddress) error { return nil } quickConfig := quic.Config{ + InitialStreamReceiveWindow: 10_000_000_000, + MaxStreamReceiveWindow: 10_000_000_000, + InitialConnectionReceiveWindow: 10_000_000_000, + MaxConnectionReceiveWindow: 10_000_000_000, + MaxIncomingStreams: 1000000000, + MaxIncomingUniStreams: 1000000000, // TODO(rach-id): do we want to enable 0RTT? are the replay risks fine? - Allow0RTT: false, - MaxIdleTimeout: time.Minute, - MaxIncomingStreams: 10000, - MaxIncomingUniStreams: 10000, - KeepAlivePeriod: 10 * time.Second, - EnableDatagrams: true, + Allow0RTT: false, + MaxIdleTimeout: time.Minute, + KeepAlivePeriod: 10 * time.Second, + EnableDatagrams: true, Tracer: func(ctx context.Context, perspective logging.Perspective, id quic.ConnectionID) *logging.ConnectionTracer { return logging.NewMultiplexedConnectionTracer(GetNewTracer()) },