From cba074e75bd3d8377f190df5c2947b65a7917918 Mon Sep 17 00:00:00 2001 From: aidan46 <47111423+aidan46@users.noreply.github.com> Date: Thu, 16 Jan 2025 11:09:34 +0100 Subject: [PATCH] feat(storage-provider-server): add p2p bootstrap and registration (#666) --- Cargo.lock | 655 ++++++++++++++++-- Cargo.toml | 2 + docs/src/storage-provider-cli/server.md | 44 +- examples/rpc_publish.sh | 31 +- examples/start_sp.sh | 15 +- pallets/market/src/mock.rs | 4 +- pallets/storage-provider/src/tests/mod.rs | 4 +- primitives/src/lib.rs | 6 + runtime/src/configs/mod.rs | 3 +- storage-provider/client/Cargo.toml | 2 + storage-provider/client/src/commands/mod.rs | 28 + storage-provider/server/Cargo.toml | 2 + storage-provider/server/src/config.rs | 31 +- storage-provider/server/src/main.rs | 120 +++- storage-provider/server/src/p2p/bootstrap.rs | 102 +++ storage-provider/server/src/p2p/mod.rs | 168 +++++ storage-provider/server/src/p2p/register.rs | 141 ++++ storage-provider/server/src/rpc.rs | 6 +- storagext/cli/Cargo.toml | 1 + storagext/cli/src/cmd/storage_provider.rs | 14 +- storagext/lib/Cargo.toml | 1 + storagext/lib/src/clients/storage_provider.rs | 7 +- 22 files changed, 1261 insertions(+), 126 deletions(-) create mode 100644 storage-provider/server/src/p2p/bootstrap.rs create mode 100644 storage-provider/server/src/p2p/mod.rs create mode 100644 storage-provider/server/src/p2p/register.rs diff --git a/Cargo.lock b/Cargo.lock index 356615726..b87ba986c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1160,6 +1160,19 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "asynchronous-codec" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a860072022177f903e59730004fb5dc13db9275b79bb2aef7ba8ce831956c233" +dependencies = [ + "bytes", + "futures-sink", + "futures-util", + "memchr", + "pin-project-lite", +] + [[package]] name = "atomic-take" version = "1.1.0" @@ -1420,6 +1433,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "bimap" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" + [[package]] name = "binary-merkle-tree" version = "13.0.0" @@ -4072,6 +4091,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" dependencies = [ "const-oid", + "pem-rfc7468", "zeroize", ] @@ -5864,6 +5884,16 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-bounded" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91f328e7fb845fc832912fb6a34f40cf6d1888c92f974d1893a54e97b5ff542e" +dependencies = [ + "futures-timer", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.31" @@ -5947,6 +5977,17 @@ dependencies = [ "rustls 0.21.12", ] +[[package]] +name = "futures-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" +dependencies = [ + "futures-io", + "rustls 0.23.20", + "rustls-pki-types", +] + [[package]] name = "futures-sink" version = "0.3.31" @@ -6038,8 +6079,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -6331,6 +6374,7 @@ dependencies = [ "ipnet", "once_cell", "rand", + "socket2 0.5.8", "thiserror 1.0.69", "tinyvec", "tokio", @@ -7822,25 +7866,57 @@ dependencies = [ "futures-timer", "getrandom", "instant", - "libp2p-allow-block-list", - "libp2p-connection-limits", - "libp2p-core", - "libp2p-dns", - "libp2p-identify", + "libp2p-allow-block-list 0.2.0", + "libp2p-connection-limits 0.2.1", + "libp2p-core 0.40.1", + "libp2p-dns 0.40.1", + "libp2p-identify 0.43.1", "libp2p-identity", "libp2p-kad", - "libp2p-mdns", - "libp2p-metrics", - "libp2p-noise", + "libp2p-mdns 0.44.0", + "libp2p-metrics 0.13.1", + "libp2p-noise 0.43.2", "libp2p-ping", - "libp2p-quic", - "libp2p-request-response", - "libp2p-swarm", - "libp2p-tcp", - "libp2p-upnp", + "libp2p-quic 0.9.3", + "libp2p-request-response 0.25.3", + "libp2p-swarm 0.43.7", + "libp2p-tcp 0.40.1", + "libp2p-upnp 0.1.1", "libp2p-wasm-ext", "libp2p-websocket", - "libp2p-yamux", + "libp2p-yamux 0.44.1", + "multiaddr 0.18.2", + "pin-project", + "rw-stream-sink", + "thiserror 1.0.69", +] + +[[package]] +name = "libp2p" +version = "0.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbe80f9c7e00526cd6b838075b9c171919404a4732cb2fa8ece0a093223bfc4" +dependencies = [ + "bytes", + "either", + "futures", + "futures-timer", + "getrandom", + "libp2p-allow-block-list 0.4.0", + "libp2p-connection-limits 0.4.0", + "libp2p-core 0.42.0", + "libp2p-dns 0.42.0", + "libp2p-identify 0.45.0", + "libp2p-identity", + "libp2p-mdns 0.46.0", + "libp2p-metrics 0.15.0", + "libp2p-noise 0.45.0", + "libp2p-quic 0.11.1", + "libp2p-rendezvous", + "libp2p-swarm 0.45.1", + "libp2p-tcp 0.42.0", + "libp2p-upnp 0.3.0", + "libp2p-yamux 0.46.0", "multiaddr 0.18.2", "pin-project", "rw-stream-sink", @@ -7853,9 +7929,21 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55b46558c5c0bf99d3e2a1a38fd54ff5476ca66dd1737b12466a1824dd219311" dependencies = [ - "libp2p-core", + "libp2p-core 0.40.1", "libp2p-identity", - "libp2p-swarm", + "libp2p-swarm 0.43.7", + "void", +] + +[[package]] +name = "libp2p-allow-block-list" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1027ccf8d70320ed77e984f273bc8ce952f623762cb9bf2d126df73caef8041" +dependencies = [ + "libp2p-core 0.42.0", + "libp2p-identity", + "libp2p-swarm 0.45.1", "void", ] @@ -7865,9 +7953,21 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f5107ad45cb20b2f6c3628c7b6014b996fcb13a88053f4569c872c6e30abf58" dependencies = [ - "libp2p-core", + "libp2p-core 0.40.1", + "libp2p-identity", + "libp2p-swarm 0.43.7", + "void", +] + +[[package]] +name = "libp2p-connection-limits" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d003540ee8baef0d254f7b6bfd79bac3ddf774662ca0abf69186d517ef82ad8" +dependencies = [ + "libp2p-core 0.42.0", "libp2p-identity", - "libp2p-swarm", + "libp2p-swarm 0.45.1", "void", ] @@ -7899,6 +7999,34 @@ dependencies = [ "void", ] +[[package]] +name = "libp2p-core" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a61f26c83ed111104cd820fe9bc3aaabbac5f1652a1d213ed6e900b7918a1298" +dependencies = [ + "either", + "fnv", + "futures", + "futures-timer", + "libp2p-identity", + "multiaddr 0.18.2", + "multihash 0.19.3", + "multistream-select", + "once_cell", + "parking_lot 0.12.3", + "pin-project", + "quick-protobuf", + "rand", + "rw-stream-sink", + "smallvec", + "thiserror 1.0.69", + "tracing", + "unsigned-varint 0.8.0", + "void", + "web-time", +] + [[package]] name = "libp2p-dns" version = "0.40.1" @@ -7907,7 +8035,7 @@ checksum = "e6a18db73084b4da2871438f6239fef35190b05023de7656e877c18a00541a3b" dependencies = [ "async-trait", "futures", - "libp2p-core", + "libp2p-core 0.40.1", "libp2p-identity", "log", "parking_lot 0.12.3", @@ -7915,29 +8043,68 @@ dependencies = [ "trust-dns-resolver", ] +[[package]] +name = "libp2p-dns" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97f37f30d5c7275db282ecd86e54f29dd2176bd3ac656f06abf43bedb21eb8bd" +dependencies = [ + "async-trait", + "futures", + "hickory-resolver", + "libp2p-core 0.42.0", + "libp2p-identity", + "parking_lot 0.12.3", + "smallvec", + "tracing", +] + [[package]] name = "libp2p-identify" version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45a96638a0a176bec0a4bcaebc1afa8cf909b114477209d7456ade52c61cd9cd" dependencies = [ - "asynchronous-codec", + "asynchronous-codec 0.6.2", "either", "futures", - "futures-bounded", + "futures-bounded 0.1.0", "futures-timer", - "libp2p-core", + "libp2p-core 0.40.1", "libp2p-identity", - "libp2p-swarm", + "libp2p-swarm 0.43.7", "log", "lru 0.12.5", "quick-protobuf", - "quick-protobuf-codec", + "quick-protobuf-codec 0.2.0", "smallvec", "thiserror 1.0.69", "void", ] +[[package]] +name = "libp2p-identify" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1711b004a273be4f30202778856368683bd9a83c4c7dcc8f848847606831a4e3" +dependencies = [ + "asynchronous-codec 0.7.0", + "either", + "futures", + "futures-bounded 0.2.4", + "futures-timer", + "libp2p-core 0.42.0", + "libp2p-identity", + "libp2p-swarm 0.45.1", + "lru 0.12.5", + "quick-protobuf", + "quick-protobuf-codec 0.3.1", + "smallvec", + "thiserror 1.0.69", + "tracing", + "void", +] + [[package]] name = "libp2p-identity" version = "0.2.10" @@ -7963,19 +8130,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16ea178dabba6dde6ffc260a8e0452ccdc8f79becf544946692fff9d412fc29d" dependencies = [ "arrayvec 0.7.6", - "asynchronous-codec", + "asynchronous-codec 0.6.2", "bytes", "either", "fnv", "futures", "futures-timer", "instant", - "libp2p-core", + "libp2p-core 0.40.1", "libp2p-identity", - "libp2p-swarm", + "libp2p-swarm 0.43.7", "log", "quick-protobuf", - "quick-protobuf-codec", + "quick-protobuf-codec 0.2.0", "rand", "sha2 0.10.8", "smallvec", @@ -7994,9 +8161,9 @@ dependencies = [ "data-encoding", "futures", "if-watch", - "libp2p-core", + "libp2p-core 0.40.1", "libp2p-identity", - "libp2p-swarm", + "libp2p-swarm 0.43.7", "log", "rand", "smallvec", @@ -8006,6 +8173,27 @@ dependencies = [ "void", ] +[[package]] +name = "libp2p-mdns" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b8546b6644032565eb29046b42744aee1e9f261ed99671b2c93fb140dba417" +dependencies = [ + "data-encoding", + "futures", + "hickory-proto", + "if-watch", + "libp2p-core 0.42.0", + "libp2p-identity", + "libp2p-swarm 0.45.1", + "rand", + "smallvec", + "socket2 0.5.8", + "tokio", + "tracing", + "void", +] + [[package]] name = "libp2p-metrics" version = "0.13.1" @@ -8013,14 +8201,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239ba7d28f8d0b5d77760dc6619c05c7e88e74ec8fbbe97f856f20a56745e620" dependencies = [ "instant", - "libp2p-core", - "libp2p-identify", + "libp2p-core 0.40.1", + "libp2p-identify 0.43.1", "libp2p-identity", "libp2p-kad", "libp2p-ping", - "libp2p-swarm", + "libp2p-swarm 0.43.7", "once_cell", - "prometheus-client", + "prometheus-client 0.21.2", +] + +[[package]] +name = "libp2p-metrics" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ebafa94a717c8442d8db8d3ae5d1c6a15e30f2d347e0cd31d057ca72e42566" +dependencies = [ + "futures", + "libp2p-core 0.42.0", + "libp2p-identify 0.45.0", + "libp2p-identity", + "libp2p-swarm 0.45.1", + "pin-project", + "prometheus-client 0.22.3", + "web-time", ] [[package]] @@ -8032,7 +8236,7 @@ dependencies = [ "bytes", "curve25519-dalek 4.1.3", "futures", - "libp2p-core", + "libp2p-core 0.40.1", "libp2p-identity", "log", "multiaddr 0.18.2", @@ -8048,6 +8252,32 @@ dependencies = [ "zeroize", ] +[[package]] +name = "libp2p-noise" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36b137cb1ae86ee39f8e5d6245a296518912014eaa87427d24e6ff58cfc1b28c" +dependencies = [ + "asynchronous-codec 0.7.0", + "bytes", + "curve25519-dalek 4.1.3", + "futures", + "libp2p-core 0.42.0", + "libp2p-identity", + "multiaddr 0.18.2", + "multihash 0.19.3", + "once_cell", + "quick-protobuf", + "rand", + "sha2 0.10.8", + "snow", + "static_assertions", + "thiserror 1.0.69", + "tracing", + "x25519-dalek", + "zeroize", +] + [[package]] name = "libp2p-ping" version = "0.43.1" @@ -8058,9 +8288,9 @@ dependencies = [ "futures", "futures-timer", "instant", - "libp2p-core", + "libp2p-core 0.40.1", "libp2p-identity", - "libp2p-swarm", + "libp2p-swarm 0.43.7", "log", "rand", "void", @@ -8076,12 +8306,12 @@ dependencies = [ "futures", "futures-timer", "if-watch", - "libp2p-core", + "libp2p-core 0.40.1", "libp2p-identity", - "libp2p-tls", + "libp2p-tls 0.2.1", "log", "parking_lot 0.12.3", - "quinn", + "quinn 0.10.2", "rand", "ring 0.16.20", "rustls 0.21.12", @@ -8090,6 +8320,54 @@ dependencies = [ "tokio", ] +[[package]] +name = "libp2p-quic" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46352ac5cd040c70e88e7ff8257a2ae2f891a4076abad2c439584a31c15fd24e" +dependencies = [ + "bytes", + "futures", + "futures-timer", + "if-watch", + "libp2p-core 0.42.0", + "libp2p-identity", + "libp2p-tls 0.5.0", + "parking_lot 0.12.3", + "quinn 0.11.6", + "rand", + "ring 0.17.8", + "rustls 0.23.20", + "socket2 0.5.8", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "libp2p-rendezvous" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b7e42a1a3315589649590228bbea53fa980e7e8e523dbe6edfbd9c1eb26de3e" +dependencies = [ + "async-trait", + "asynchronous-codec 0.7.0", + "bimap", + "futures", + "futures-timer", + "libp2p-core 0.42.0", + "libp2p-identity", + "libp2p-request-response 0.27.0", + "libp2p-swarm 0.45.1", + "quick-protobuf", + "quick-protobuf-codec 0.3.1", + "rand", + "thiserror 1.0.69", + "tracing", + "void", + "web-time", +] + [[package]] name = "libp2p-request-response" version = "0.25.3" @@ -8099,15 +8377,35 @@ dependencies = [ "async-trait", "futures", "instant", - "libp2p-core", + "libp2p-core 0.40.1", "libp2p-identity", - "libp2p-swarm", + "libp2p-swarm 0.43.7", "log", "rand", "smallvec", "void", ] +[[package]] +name = "libp2p-request-response" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1356c9e376a94a75ae830c42cdaea3d4fe1290ba409a22c809033d1b7dcab0a6" +dependencies = [ + "async-trait", + "futures", + "futures-bounded 0.2.4", + "futures-timer", + "libp2p-core 0.42.0", + "libp2p-identity", + "libp2p-swarm 0.45.1", + "rand", + "smallvec", + "tracing", + "void", + "web-time", +] + [[package]] name = "libp2p-swarm" version = "0.43.7" @@ -8119,9 +8417,9 @@ dependencies = [ "futures", "futures-timer", "instant", - "libp2p-core", + "libp2p-core 0.40.1", "libp2p-identity", - "libp2p-swarm-derive", + "libp2p-swarm-derive 0.33.0", "log", "multistream-select", "once_cell", @@ -8131,6 +8429,30 @@ dependencies = [ "void", ] +[[package]] +name = "libp2p-swarm" +version = "0.45.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7dd6741793d2c1fb2088f67f82cf07261f25272ebe3c0b0c311e0c6b50e851a" +dependencies = [ + "either", + "fnv", + "futures", + "futures-timer", + "libp2p-core 0.42.0", + "libp2p-identity", + "libp2p-swarm-derive 0.35.0", + "lru 0.12.5", + "multistream-select", + "once_cell", + "rand", + "smallvec", + "tokio", + "tracing", + "void", + "web-time", +] + [[package]] name = "libp2p-swarm-derive" version = "0.33.0" @@ -8144,6 +8466,18 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "libp2p-swarm-derive" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206e0aa0ebe004d778d79fb0966aa0de996c19894e2c0605ba2f8524dd4443d8" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "libp2p-tcp" version = "0.40.1" @@ -8154,13 +8488,30 @@ dependencies = [ "futures-timer", "if-watch", "libc", - "libp2p-core", + "libp2p-core 0.40.1", "libp2p-identity", "log", "socket2 0.5.8", "tokio", ] +[[package]] +name = "libp2p-tcp" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad964f312c59dcfcac840acd8c555de8403e295d39edf96f5240048b5fcaa314" +dependencies = [ + "futures", + "futures-timer", + "if-watch", + "libc", + "libp2p-core 0.42.0", + "libp2p-identity", + "socket2 0.5.8", + "tokio", + "tracing", +] + [[package]] name = "libp2p-tls" version = "0.2.1" @@ -8168,10 +8519,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8218d1d5482b122ccae396bbf38abdcb283ecc96fa54760e1dfd251f0546ac61" dependencies = [ "futures", - "futures-rustls", - "libp2p-core", + "futures-rustls 0.24.0", + "libp2p-core 0.40.1", "libp2p-identity", - "rcgen", + "rcgen 0.10.0", "ring 0.16.20", "rustls 0.21.12", "rustls-webpki 0.101.7", @@ -8180,6 +8531,25 @@ dependencies = [ "yasna", ] +[[package]] +name = "libp2p-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b23dddc2b9c355f73c1e36eb0c3ae86f7dc964a3715f0731cfad352db4d847" +dependencies = [ + "futures", + "futures-rustls 0.26.0", + "libp2p-core 0.42.0", + "libp2p-identity", + "rcgen 0.11.3", + "ring 0.17.8", + "rustls 0.23.20", + "rustls-webpki 0.101.7", + "thiserror 1.0.69", + "x509-parser 0.16.0", + "yasna", +] + [[package]] name = "libp2p-upnp" version = "0.1.1" @@ -8189,13 +8559,29 @@ dependencies = [ "futures", "futures-timer", "igd-next", - "libp2p-core", - "libp2p-swarm", + "libp2p-core 0.40.1", + "libp2p-swarm 0.43.7", "log", "tokio", "void", ] +[[package]] +name = "libp2p-upnp" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01bf2d1b772bd3abca049214a3304615e6a36fa6ffc742bdd1ba774486200b8f" +dependencies = [ + "futures", + "futures-timer", + "igd-next", + "libp2p-core 0.42.0", + "libp2p-swarm 0.45.1", + "tokio", + "tracing", + "void", +] + [[package]] name = "libp2p-wasm-ext" version = "0.40.0" @@ -8204,7 +8590,7 @@ checksum = "1e5d8e3a9e07da0ef5b55a9f26c009c8fb3c725d492d8bb4b431715786eea79c" dependencies = [ "futures", "js-sys", - "libp2p-core", + "libp2p-core 0.40.1", "send_wrapper", "wasm-bindgen", "wasm-bindgen-futures", @@ -8218,8 +8604,8 @@ checksum = "004ee9c4a4631435169aee6aad2f62e3984dc031c43b6d29731e8e82a016c538" dependencies = [ "either", "futures", - "futures-rustls", - "libp2p-core", + "futures-rustls 0.24.0", + "libp2p-core 0.40.1", "libp2p-identity", "log", "parking_lot 0.12.3", @@ -8238,10 +8624,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eedcb62824c4300efb9cfd4e2a6edaf3ca097b9e68b36dabe45a44469fd6a85" dependencies = [ "futures", - "libp2p-core", + "libp2p-core 0.40.1", "log", "thiserror 1.0.69", - "yamux", + "yamux 0.12.1", +] + +[[package]] +name = "libp2p-yamux" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "788b61c80789dba9760d8c669a5bedb642c8267555c803fabd8396e4ca5c5882" +dependencies = [ + "either", + "futures", + "libp2p-core 0.42.0", + "thiserror 1.0.69", + "tracing", + "yamux 0.12.1", + "yamux 0.13.4", ] [[package]] @@ -8427,7 +8828,7 @@ dependencies = [ "prost 0.12.6", "prost-build", "rand", - "rcgen", + "rcgen 0.10.0", "ring 0.16.20", "rustls 0.20.9", "serde", @@ -13146,6 +13547,15 @@ dependencies = [ "serde", ] +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -13408,8 +13818,10 @@ dependencies = [ "bls12_381", "cid 0.11.1", "clap", + "ed25519-dalek", "hex", "jsonrpsee 0.24.7", + "libp2p 0.54.1", "mater", "parity-scale-codec", "polka-storage-proofs", @@ -13454,10 +13866,12 @@ dependencies = [ "axum", "cid 0.11.1", "clap", + "ed25519-dalek", "futures", "hex", "hyper 1.5.2", "jsonrpsee 0.24.7", + "libp2p 0.54.1", "mater", "parity-scale-codec", "polka-storage-proofs", @@ -15704,6 +16118,18 @@ dependencies = [ "prometheus-client-derive-encode", ] +[[package]] +name = "prometheus-client" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "504ee9ff529add891127c4827eb481bd69dc0ebc72e9a682e187db4caa60c3ca" +dependencies = [ + "dtoa", + "itoa", + "parking_lot 0.12.3", + "prometheus-client-derive-encode", +] + [[package]] name = "prometheus-client-derive-encode" version = "0.4.2" @@ -15855,13 +16281,26 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ededb1cd78531627244d51dd0c7139fbe736c7d57af0092a76f0ffb2f56e98" dependencies = [ - "asynchronous-codec", + "asynchronous-codec 0.6.2", "bytes", "quick-protobuf", "thiserror 1.0.69", "unsigned-varint 0.7.2", ] +[[package]] +name = "quick-protobuf-codec" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15a0580ab32b169745d7a39db2ba969226ca16738931be152a3209b409de2474" +dependencies = [ + "asynchronous-codec 0.7.0", + "bytes", + "quick-protobuf", + "thiserror 1.0.69", + "unsigned-varint 0.8.0", +] + [[package]] name = "quinn" version = "0.10.2" @@ -15871,8 +16310,8 @@ dependencies = [ "bytes", "futures-io", "pin-project-lite", - "quinn-proto", - "quinn-udp", + "quinn-proto 0.10.6", + "quinn-udp 0.4.1", "rustc-hash 1.1.0", "rustls 0.21.12", "thiserror 1.0.69", @@ -15880,6 +16319,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "quinn" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" +dependencies = [ + "bytes", + "futures-io", + "pin-project-lite", + "quinn-proto 0.11.9", + "quinn-udp 0.5.9", + "rustc-hash 2.1.0", + "rustls 0.23.20", + "socket2 0.5.8", + "thiserror 2.0.8", + "tokio", + "tracing", +] + [[package]] name = "quinn-proto" version = "0.10.6" @@ -15897,6 +16355,26 @@ dependencies = [ "tracing", ] +[[package]] +name = "quinn-proto" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" +dependencies = [ + "bytes", + "getrandom", + "rand", + "ring 0.17.8", + "rustc-hash 2.1.0", + "rustls 0.23.20", + "rustls-pki-types", + "slab", + "thiserror 2.0.8", + "tinyvec", + "tracing", + "web-time", +] + [[package]] name = "quinn-udp" version = "0.4.1" @@ -15910,6 +16388,20 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "quinn-udp" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" +dependencies = [ + "cfg_aliases 0.2.1", + "libc", + "once_cell", + "socket2 0.5.8", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.37" @@ -16036,6 +16528,18 @@ dependencies = [ "yasna", ] +[[package]] +name = "rcgen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c4f3084aa3bc7dfbba4eff4fab2a54db4324965d8872ab933565e6fbd83bc6" +dependencies = [ + "pem 3.0.4", + "ring 0.16.20", + "time", + "yasna", +] + [[package]] name = "reconnecting-jsonrpsee-ws-client" version = "0.4.3" @@ -16795,6 +17299,9 @@ name = "rustls-pki-types" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" +dependencies = [ + "web-time", +] [[package]] name = "rustls-platform-verifier" @@ -16979,7 +17486,7 @@ dependencies = [ "futures", "futures-timer", "ip_network", - "libp2p", + "libp2p 0.52.4", "linked_hash_set", "log", "multihash 0.19.3", @@ -17639,7 +18146,7 @@ dependencies = [ "array-bytes", "async-channel 1.9.0", "async-trait", - "asynchronous-codec", + "asynchronous-codec 0.6.2", "bytes", "cid 0.9.0", "either", @@ -17647,7 +18154,7 @@ dependencies = [ "futures", "futures-timer", "ip_network", - "libp2p", + "libp2p 0.52.4", "linked_hash_set", "litep2p", "log", @@ -18101,7 +18608,7 @@ source = "git+https://github.com/paritytech/polkadot-sdk?tag=polkadot-stable2412 dependencies = [ "chrono", "futures", - "libp2p", + "libp2p 0.52.4", "log", "parking_lot 0.12.3", "pin-project", @@ -21914,6 +22421,7 @@ dependencies = [ "futures", "hex", "itertools 0.13.0", + "libp2p 0.54.1", "parity-scale-codec", "primitives", "serde", @@ -21937,6 +22445,7 @@ dependencies = [ "clap", "frame-support 28.0.0", "hex", + "libp2p 0.54.1", "parity-scale-codec", "primitives", "serde", @@ -23553,7 +24062,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" dependencies = [ - "asynchronous-codec", + "asynchronous-codec 0.6.2", "bytes", "futures-io", "futures-util", @@ -24850,6 +25359,22 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "yamux" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17610762a1207ee816c6fadc29220904753648aba0a9ed61c7b8336e80a559c4" +dependencies = [ + "futures", + "log", + "nohash-hasher", + "parking_lot 0.12.3", + "pin-project", + "rand", + "static_assertions", + "web-time", +] + [[package]] name = "yansi" version = "1.0.1" @@ -25020,7 +25545,7 @@ dependencies = [ "futures", "glob-match", "hex", - "libp2p", + "libp2p 0.52.4", "libsecp256k1", "multiaddr 0.18.2", "rand", diff --git a/Cargo.toml b/Cargo.toml index d69a879d1..025d80b8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ color-print = "0.3.4" criterion = "0.5.1" digest = "0.10.7" docify = { version = "0.2.8" } +ed25519-dalek = { version = "2.1.1", default-features = false } env_logger = "0.11.2" ff = "0.13.0" futures = "0.3.28" @@ -79,6 +80,7 @@ ipld-core = "0.4.1" ipld-dagpb = "0.2.1" itertools = "0.13.0" jsonrpsee = { version = "0.24.7" } +libp2p = { version = "0.54", default-features = false } log = { version = "0.4.21", default-features = false } multihash-codetable = { version = "0.1.1", default-features = false } num-bigint = { version = "0.4.5", default-features = false } diff --git a/docs/src/storage-provider-cli/server.md b/docs/src/storage-provider-cli/server.md index 1a4ae3ab5..e3febfde7 100644 --- a/docs/src/storage-provider-cli/server.md +++ b/docs/src/storage-provider-cli/server.md @@ -81,22 +81,42 @@ The path to the PoRep proving parameters. They are shared across all of the node The path to the PoSt proving parameters. They are shared across all of the nodes in the network, as the chain stores corresponding Verifying Key parameters. +### `--node-type` + +The P2P node type that the server will be running. Can be either `bootstrap` or `registration` node. + +### `--p2p-key` + +The ED25519 private key used by the P2P node. + +### `--rendezvous-point-address` + +Rendezvous point address that the registration node connects to or the bootstrap node binds to. + +### `--rendezvous-point` + +Peer ID of the rendezvous point that the registration node connects to. Only needed if running a registration P2P node. + ### `--config` Takes in a path to a configuration file, it supports both JSON and TOML (files _must_ have the right extension). The supported configuration parameters are: -| Name | Default | -| ----------------------- | ------------------------------ | -| `upload-listen-address` | `127.0.0.1:8000` | -| `rpc-listen-address` | `127.0.0.1:8001` | -| `node-url` | `ws://127.0.0.1:42069` | -| `database-directory` | `/tmp//deals_database` | -| `storage-directory` | `/tmp//deals_storage` | -| `seal-proof` | 2KiB | -| `post-proof` | 2KiB | -| `porep_parameters` | NA | -| `post_parameters` | NA | +| Name | Default | +| -------------------------- | ------------------------------ | +| `upload-listen-address` | `127.0.0.1:8000` | +| `rpc-listen-address` | `127.0.0.1:8001` | +| `node-url` | `ws://127.0.0.1:42069` | +| `database-directory` | `/tmp//deals_database` | +| `storage-directory` | `/tmp//deals_storage` | +| `seal-proof` | `2KiB` | +| `post-proof` | `2KiB` | +| `porep_parameters` | NA | +| `post_parameters` | NA | +| `node_type` | `bootstrap` | +| `p2p_key` | NA | +| `rendezvous_point_address` | NA | +| `rendezvous_point` | `None` | #### Bare bones configuration @@ -104,5 +124,7 @@ The supported configuration parameters are: { "porep_parameters": "/home/storage_provider/porep.params", "post_parameters": "/home/storage_provider/post.params", + "p2p_key": "/home/storage_provider/private_key.pem", + "rendezvous_point_address": "/ip4/127.0.0.1/tcp/62649", } ``` diff --git a/examples/rpc_publish.sh b/examples/rpc_publish.sh index d22693e42..8f9f2992f 100755 --- a/examples/rpc_publish.sh +++ b/examples/rpc_publish.sh @@ -21,13 +21,38 @@ PROVIDER="//Charlie" INPUT_FILE="$1" INPUT_FILE_NAME="$(basename "$INPUT_FILE")" +# CARv2 file location INPUT_TMP_FILE="/tmp/$INPUT_FILE_NAME.car" - +# Config file location +CONFIG="/tmp/config.toml" +# P2P Node variables +P2P_PUBLIC_KEY="/tmp/public.pem" +P2P_PRIVATE_KEY="/tmp/private.pem" +P2P_ADDRESS="/ip4/127.0.0.1/tcp/62649" + +# Generate ED25519 private key to be replaced with a polka-storage-provider-client command +# TODO(@aidan46, 15/01, #676) +openssl genpkey -algorithm ED25519 -out "$P2P_PRIVATE_KEY" -outpubkey "$P2P_PUBLIC_KEY" + +# Convert file to CARv2 format target/release/mater-cli convert -q --overwrite "$INPUT_FILE" "$INPUT_TMP_FILE" && + +# Calculate COMMP and set PIECE_CID and PIECE_SIZE INPUT_COMMP="$(target/release/polka-storage-provider-client proofs commp "$INPUT_TMP_FILE")" PIECE_CID="$(echo "$INPUT_COMMP" | jq -r ".cid")" PIECE_SIZE="$(echo "$INPUT_COMMP" | jq ".size")" +# Generate Peer ID from public key +PEER_ID="$(target/release/polka-storage-provider-client generate-peer-id --pubkey "$P2P_PUBLIC_KEY")" + +# echo config file in the file in the /tmp folder +echo "seal_proof = '2KiB' +post_proof = '2KiB' +porep_parameters = '2KiB.porep.params' +post_parameters = '2KiB.post.params' +rendezvous_point_address = '$P2P_ADDRESS' +p2p_key = '@$P2P_PRIVATE_KEY'" > "$CONFIG" + # Setup balances target/release/storagext-cli --sr25519-key "$CLIENT" market add-balance 250000000000 & @@ -39,7 +64,7 @@ wait # It's a test setup based on the local verifying keys, everyone can run those extrinsics currently. # Each of the keys is different, because the processes are running in parallel. # If they were running in parallel on the same account, they'd conflict with each other on the transaction nonce. -target/release/storagext-cli --sr25519-key "//Charlie" storage-provider register "peer_id" & +target/release/storagext-cli --sr25519-key "//Charlie" storage-provider register "$PEER_ID" & target/release/storagext-cli --sr25519-key "//Alice" proofs set-porep-verifying-key @2KiB.porep.vk.scale & target/release/storagext-cli --sr25519-key "//Bob" proofs set-post-verifying-key @2KiB.post.vk.scale & @@ -64,7 +89,7 @@ DEAL_JSON=$( ) SIGNED_DEAL_JSON="$(RUST_LOG=error target/release/polka-storage-provider-client sign-deal --sr25519-key "$CLIENT" "$DEAL_JSON")" -(RUST_LOG=debug target/release/polka-storage-provider-server --sr25519-key "$PROVIDER" --seal-proof "2KiB" --post-proof "2KiB" --porep-parameters 2KiB.porep.params --post-parameters 2KiB.post.params) & +(RUST_LOG=debug target/release/polka-storage-provider-server --sr25519-key "$PROVIDER" --config "$CONFIG") & sleep 5 # gives time for the server to start DEAL_CID="$(RUST_LOG=error target/release/polka-storage-provider-client propose-deal "$DEAL_JSON")" diff --git a/examples/start_sp.sh b/examples/start_sp.sh index 47f48f416..4d10421e3 100755 --- a/examples/start_sp.sh +++ b/examples/start_sp.sh @@ -8,6 +8,15 @@ export DISABLE_XT_WAIT_WARNING=1 CLIENT="//Alice" PROVIDER="//Charlie" +P2P_PUBLIC_KEY="/tmp/public.pem" +P2P_PRIVATE_KEY="/tmp/private.pem" +P2P_ADDRESS="/ip4/127.0.0.1/tcp/62649" + +# Generate ED25519 private key +openssl genpkey -algorithm ED25519 -out "$P2P_PRIVATE_KEY" -outpubkey "$P2P_PUBLIC_KEY" + +# Generate Peer ID +PEER_ID="$(target/release/polka-storage-provider-client generate-peer-id --pubkey "$P2P_PUBLIC_KEY")" # Setup balances RUST_LOG=debug target/release/storagext-cli --sr25519-key "$CLIENT" market add-balance 250000000000 & @@ -19,7 +28,7 @@ wait # It's a test setup based on the local verifying keys, everyone can run those extrinsics currently. # Each of the keys is different, because the processes are running in parallel. # If they were running in parallel on the same account, they'd conflict with each other on the transaction nonce. -RUST_LOG=debug target/release/storagext-cli --sr25519-key "//Charlie" storage-provider register "peer_id" & +RUST_LOG=debug target/release/storagext-cli --sr25519-key "//Charlie" storage-provider register "$PEER_ID" & RUST_LOG=debug target/release/storagext-cli --sr25519-key "//Alice" proofs set-porep-verifying-key @2KiB.porep.vk.scale & RUST_LOG=debug target/release/storagext-cli --sr25519-key "//Bob" proofs set-post-verifying-key @2KiB.post.vk.scale & wait @@ -28,7 +37,9 @@ echo '{ "seal_proof": "2KiB", "post_proof": "2KiB", "porep_parameters": "2KiB.porep.params", - "post_parameters": "2KiB.post.params" + "post_parameters": "2KiB.post.params", + "p2p_key": "@/tmp/private.pem", + "rendezvous_point_address": "/ip4/127.0.0.1/tcp/62649" }' > /tmp/storage_provider.config.json RUST_LOG=debug target/release/polka-storage-provider-server \ --sr25519-key "$PROVIDER" \ diff --git a/pallets/market/src/mock.rs b/pallets/market/src/mock.rs index 00dc12f14..61fcd1803 100644 --- a/pallets/market/src/mock.rs +++ b/pallets/market/src/mock.rs @@ -6,7 +6,7 @@ use frame_support::{ PalletId, }; use frame_system::pallet_prelude::BlockNumberFor; -use primitives::proofs::RegisteredPoStProof; +use primitives::{proofs::RegisteredPoStProof, PEER_ID_MAX_BYTES}; use sp_core::Pair; use sp_runtime::{ traits::{ConstU32, ConstU64, IdentifyAccount, IdentityLookup, Verify, Zero}, @@ -119,7 +119,7 @@ impl pallet_storage_provider::Config for Test { type RuntimeEvent = RuntimeEvent; type Randomness = DummyRandomnessGenerator; type AuthorVrfHistory = DummyRandomnessGenerator; - type PeerId = BoundedVec>; // Max length of SHA256 hash + type PeerId = BoundedVec>; // https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#peer-ids type Currency = Balances; type Market = Market; type ProofVerification = Proofs; diff --git a/pallets/storage-provider/src/tests/mod.rs b/pallets/storage-provider/src/tests/mod.rs index e5e36653a..f75baea8c 100644 --- a/pallets/storage-provider/src/tests/mod.rs +++ b/pallets/storage-provider/src/tests/mod.rs @@ -21,7 +21,7 @@ use primitives::{ sector::SectorNumber, DealId, PartitionNumber, CID_SIZE_IN_BYTES, MAX_DEALS_PER_SECTOR, MAX_PARTITIONS_PER_DEADLINE, MAX_POST_PROOF_BYTES, MAX_PROOFS_PER_BLOCK, MAX_REPLICAS_PER_BLOCK, MAX_SEAL_PROOF_BYTES, - MAX_TERMINATIONS_PER_CALL, + MAX_TERMINATIONS_PER_CALL, PEER_ID_MAX_BYTES, }; use sp_arithmetic::traits::Zero; use sp_core::{bounded_vec, Pair}; @@ -198,7 +198,7 @@ impl pallet_storage_provider::Config for Test { type RuntimeEvent = RuntimeEvent; type Randomness = DummyRandomnessGenerator; type AuthorVrfHistory = DummyRandomnessGenerator; - type PeerId = BoundedVec>; // Max length of SHA256 hash + type PeerId = BoundedVec>; // https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#peer-ids type Currency = Balances; type Market = Market; type ProofVerification = DummyProofsVerification; diff --git a/primitives/src/lib.rs b/primitives/src/lib.rs index da066eccc..b2436d14b 100644 --- a/primitives/src/lib.rs +++ b/primitives/src/lib.rs @@ -76,3 +76,9 @@ pub const MAX_SEAL_PROOF_BYTES: u32 = 1_920; /// References: /// * pub const MAX_POST_PROOF_BYTES: u32 = 192; + +/// The maximum amount of bytes of a libp2p Peer ID. +/// +/// References: +/// * https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#peer-ids +pub const PEER_ID_MAX_BYTES: u32 = 42; diff --git a/runtime/src/configs/mod.rs b/runtime/src/configs/mod.rs index 712424055..fdc358615 100644 --- a/runtime/src/configs/mod.rs +++ b/runtime/src/configs/mod.rs @@ -45,6 +45,7 @@ use parachains_common::message_queue::{NarrowOriginToSibling, ParaIdToSibling}; use polkadot_runtime_common::{ xcm_sender::NoPriceForMessageDelivery, BlockHashCount, SlowAdjustingFeeUpdate, }; +use primitives::PEER_ID_MAX_BYTES; use sp_consensus_aura::sr25519::AuthorityId as AuraId; use sp_runtime::{traits::Verify, MultiSignature, Perbill}; use sp_version::RuntimeVersion; @@ -368,7 +369,7 @@ impl pallet_storage_provider::Config for Runtime { type RuntimeEvent = RuntimeEvent; type Randomness = crate::Randomness; type AuthorVrfHistory = crate::Randomness; - type PeerId = BoundedVec>; // Max length of SHA256 hash + type PeerId = BoundedVec>; // https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#peer-ids type Currency = Balances; type Market = crate::Market; type ProofVerification = crate::Proofs; diff --git a/storage-provider/client/Cargo.toml b/storage-provider/client/Cargo.toml index bc78d0d09..c3b8a9ec3 100644 --- a/storage-provider/client/Cargo.toml +++ b/storage-provider/client/Cargo.toml @@ -20,8 +20,10 @@ bls12_381 = { workspace = true } cid = { workspace = true, features = ["std"] } clap = { workspace = true, features = ["derive"] } codec = { workspace = true } +ed25519-dalek = { workspace = true, features = ["pem", "pkcs8"] } hex = { workspace = true } jsonrpsee = { workspace = true, features = ["http-client", "macros", "server", "ws-client"] } +libp2p.workspace = true sc-cli = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/storage-provider/client/src/commands/mod.rs b/storage-provider/client/src/commands/mod.rs index a6dd9b0d0..96b808b1e 100644 --- a/storage-provider/client/src/commands/mod.rs +++ b/storage-provider/client/src/commands/mod.rs @@ -1,8 +1,12 @@ mod proofs; mod wallet; +use std::path::PathBuf; + use clap::Parser; +use ed25519_dalek::pkcs8::{DecodePublicKey, PublicKeyBytes}; use jsonrpsee::core::ClientError; +use libp2p::{identity::ed25519::PublicKey as EdPubKey, PeerId}; use polka_storage_provider_common::rpc::StorageProviderRpcClient; use storagext::{ deser::DeserializablePath, @@ -47,6 +51,12 @@ pub enum CliError { #[error("no signer key was provider")] NoSigner, + + #[error(transparent)] + PubKeyError(#[from] ed25519_dalek::pkcs8::spki::Error), + + #[error(transparent)] + DecodingError(#[from] libp2p::identity::DecodingError), } /// A CLI application that facilitates management operations over a running full @@ -98,6 +108,13 @@ pub(crate) enum Cli { #[command(flatten)] signer_key: MultiPairArgs, }, + + /// Generate a Peer ID from a ED25519 public key pem file + GeneratePeerID { + /// Path to the ED25519 public key pem file + #[arg(long)] + pubkey: PathBuf, + }, } impl Cli { @@ -131,6 +148,7 @@ impl Cli { deal_proposal, signer_key, } => Self::sign_deal(deal_proposal, signer_key), + Self::GeneratePeerID { pubkey } => Self::generate_peer_id(pubkey), } } @@ -182,4 +200,14 @@ impl Cli { ); Ok(()) } + + fn generate_peer_id(path: PathBuf) -> Result<(), CliError> { + let pubkey_bytes = PublicKeyBytes::read_public_key_pem_file(path)?; + let pubkey = EdPubKey::try_from_bytes(&pubkey_bytes.to_bytes())?; + let key = libp2p::identity::PublicKey::from(pubkey); + let peer_id = PeerId::from_public_key(&key); + + println!("{peer_id}"); + Ok(()) + } } diff --git a/storage-provider/server/Cargo.toml b/storage-provider/server/Cargo.toml index 9409f9afa..820d185dd 100644 --- a/storage-provider/server/Cargo.toml +++ b/storage-provider/server/Cargo.toml @@ -24,10 +24,12 @@ axum = { workspace = true, features = ["macros", "multipart"] } cid = { workspace = true, features = ["serde", "std"] } clap = { workspace = true, features = ["derive"] } codec = { workspace = true } +ed25519-dalek = { workspace = true, features = ["pem", "std"] } futures = { workspace = true } hex = { workspace = true, features = ["std"] } hyper = { workspace = true } jsonrpsee = { workspace = true, features = ["http-client", "macros", "server", "ws-client"] } +libp2p = { workspace = true, features = ["identify", "macros", "noise", "rendezvous", "tcp", "tokio", "yamux"] } rand = { workspace = true } rocksdb = { workspace = true } sc-cli = { workspace = true } diff --git a/storage-provider/server/src/config.rs b/storage-provider/server/src/config.rs index 1e71eceeb..9b72ae5c0 100644 --- a/storage-provider/server/src/config.rs +++ b/storage-provider/server/src/config.rs @@ -5,11 +5,15 @@ use std::{ }; use clap::Args; +use libp2p::{identity::Keypair, Multiaddr, PeerId}; use primitives::proofs::{RegisteredPoStProof, RegisteredSealProof}; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use url::Url; -use crate::DEFAULT_NODE_ADDRESS; +use crate::{ + p2p::{deser_keypair, keypair_value_parser, string_to_peer_id_option, NodeType}, + DEFAULT_NODE_ADDRESS, +}; /// Default address to bind the RPC server to. const fn default_rpc_listen_address() -> SocketAddr { @@ -31,7 +35,7 @@ fn default_node_address() -> Url { Url::parse(DEFAULT_NODE_ADDRESS).expect("DEFAULT_NODE_ADDRESS must be a valid Url") } -#[derive(Debug, Clone, Deserialize, Serialize, Args)] +#[derive(Debug, Clone, Deserialize, Args)] #[group(multiple = true, conflicts_with = "config")] #[serde(deny_unknown_fields)] pub struct ConfigurationArgs { @@ -98,4 +102,25 @@ pub struct ConfigurationArgs { /// **they need to be set** via an extrinsic pallet-proofs::set_post_verifyingkey. #[arg(long, required = false)] pub(crate) post_parameters: PathBuf, + + /// P2P Node type, can be either a bootstrap node or a registration node. + #[serde(default = "NodeType::default")] + #[arg(long, default_value_t = NodeType::Bootstrap)] + pub(crate) node_type: NodeType, + + /// P2P ED25519 private key + #[serde(deserialize_with = "deser_keypair")] + #[arg(long, value_parser = keypair_value_parser)] + pub(crate) p2p_key: Keypair, + + /// Rendezvous point address that the registration node connects to + /// or the bootstrap node binds to. + #[arg(long)] + pub(crate) rendezvous_point_address: Multiaddr, + + /// PeerID of the bootstrap node used by the registration node. + /// Optional because it is not used by the bootstrap node. + #[serde(default, deserialize_with = "string_to_peer_id_option")] + #[arg(long)] + pub(crate) rendezvous_point: Option, } diff --git a/storage-provider/server/src/main.rs b/storage-provider/server/src/main.rs index 34170ff36..388d841af 100644 --- a/storage-provider/server/src/main.rs +++ b/storage-provider/server/src/main.rs @@ -4,6 +4,7 @@ mod config; mod db; +mod p2p; mod pipeline; mod rpc; mod storage; @@ -11,7 +12,11 @@ mod storage; use std::{env::temp_dir, net::SocketAddr, path::PathBuf, sync::Arc, time::Duration}; use clap::Parser; -use config::ConfigurationArgs; +use libp2p::{identity::Keypair, Multiaddr, PeerId}; +use p2p::{ + run_bootstrap_node, run_register_node, BootstrapConfig, NodeType, P2PError, P2PState, + RegisterConfig, +}; use pipeline::types::PipelineMessage; use polka_storage_proofs::{ porep::{self, PoRepParameters}, @@ -31,7 +36,7 @@ use storagext::{ use subxt::{self, tx::Signer}; use tokio::{ sync::{mpsc::UnboundedReceiver, Semaphore}, - task::JoinError, + task::{JoinError, JoinHandle}, }; use tokio_util::sync::CancellationToken; use tracing::level_filters::LevelFilter; @@ -39,6 +44,7 @@ use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt, Env use url::Url; use crate::{ + config::ConfigurationArgs, db::{DBError, DealDB}, pipeline::{start_pipeline, PipelineState}, rpc::{start_rpc_server, RpcServerState}, @@ -81,7 +87,9 @@ struct SetupOutput { rpc_state: RpcServerState, pipeline_state: PipelineState, pipeline_rx: UnboundedReceiver, + p2p_state: P2PState, } + fn main() -> Result<(), ServerError> { // Logger initialization. let file_appender = tracing_appender::rolling::daily("logs", "sp_server"); @@ -165,6 +173,33 @@ pub enum ServerError { #[error(transparent)] Json(#[from] serde_json::Error), + + #[error(transparent)] + P2P(#[from] P2PError), +} + +/// Takes an expression that returns a `Result, E1>`. +/// It tries to inspect and log the first error (`E1`), otherwise, +/// it inspects the result and tries to inspect the nested error (`E2`). +/// +/// This macro is *roughly* equivalent to calling: +/// ```text +/// res // : Result, E1> +/// .inspect_err(|e| tracing::error!(%e)) +/// .inspect(|r| r.inspect_err(|e| tracing::error!(%e)) +/// ``` +macro_rules! inspect_and_log_nested_errors { + ($($task:expr),+ $(,)?) => { + ( + $( + $task + .inspect_err(|err| tracing::error!(%err)) + .inspect(|ok| { + let _ = ok.as_ref().inspect_err(|err| tracing::error!(%err)); + }) + ),+ + ) + }; } /// The server arguments, as passed by the user, unvalidated. @@ -223,6 +258,20 @@ pub struct Server { /// The number of prove commits to be run in parallel. parallel_prove_commits: usize, + + /// P2P Network node type, can either be a bootstrap or registration node + node_type: NodeType, + + /// P2P ED25519 private key + p2p_key: Keypair, + + /// Rendezvous point address that the registration node connects to + /// or the bootstrap node binds to. + rendezvous_point_address: Multiaddr, + + /// PeerID of the bootstrap node used by the registration node. + /// Optional because it is not used by the bootstrap node. + rendezvous_point: Option, } impl TryFrom for Server { @@ -300,6 +349,10 @@ impl TryFrom for Server { porep_parameters, post_parameters, parallel_prove_commits: args.parallel_prove_commits.get(), + node_type: args.node_type, + p2p_key: args.p2p_key, + rendezvous_point_address: args.rendezvous_point_address, + rendezvous_point: args.rendezvous_point, }) } } @@ -311,10 +364,12 @@ impl Server { rpc_state, pipeline_state, pipeline_rx, + p2p_state, } = self.setup().await?; let cancellation_token = CancellationToken::new(); + let p2p_task = spawn_p2p_task(p2p_state, cancellation_token.child_token())?; let rpc_task = tokio::spawn(start_rpc_server( rpc_state, cancellation_token.child_token(), @@ -339,31 +394,18 @@ impl Server { tracing::info!("sent shutdown signal"); // Wait for the tasks to finish - let (upload_result, rpc_task, pipeline_task) = - tokio::join!(storage_task, rpc_task, pipeline_task); - - // Log errors - let upload_result = upload_result - .inspect_err(|err| tracing::error!(%err)) - .inspect(|ok| { - let _ = ok.as_ref().inspect_err(|err| tracing::error!(%err)); - }); - let rpc_task = rpc_task - .inspect_err(|err| tracing::error!(%err)) - .inspect(|ok| { - let _ = ok.as_ref().inspect_err(|err| tracing::error!(%err)); - }); - - let pipeline_task = pipeline_task - .inspect_err(|err| tracing::error!(%err)) - .inspect(|ok| { - let _ = ok.as_ref().inspect_err(|err| tracing::error!(%err)); - }); + let (upload_result, rpc_task, pipeline_task, p2p_task) = + tokio::join!(storage_task, rpc_task, pipeline_task, p2p_task); + + // Inspect and log errors + let (upload_result, rpc_task, pipeline_task, p2p_task) = + inspect_and_log_nested_errors!(upload_result, rpc_task, pipeline_task, p2p_task); // Exit with error upload_result??; rpc_task??; pipeline_task??; + p2p_task??; Ok(()) } @@ -434,11 +476,19 @@ impl Server { prove_commit_throttle: Arc::new(Semaphore::new(self.parallel_prove_commits)), }; + let p2p_state = P2PState { + node_type: self.node_type, + p2p_key: self.p2p_key, + rendezvous_point_address: self.rendezvous_point_address, + rendezvous_point: self.rendezvous_point, + }; + Ok(SetupOutput { storage_state, rpc_state, pipeline_state, pipeline_rx, + p2p_state, }) } @@ -497,3 +547,29 @@ impl Server { } } } + +/// Spawns a p2p node and returns a `JoinHandle`. +/// The node type is either bootstrap or registration depending on the `p2p_state.node_type` value. +fn spawn_p2p_task( + p2p_state: P2PState, + cancellation_token: CancellationToken, +) -> Result>, ServerError> { + match p2p_state.node_type { + NodeType::Bootstrap => { + let config = + BootstrapConfig::new(p2p_state.p2p_key, p2p_state.rendezvous_point_address); + Ok(tokio::spawn(run_bootstrap_node(config, cancellation_token))) + } + NodeType::Register => { + let Some(rendezvous_point) = p2p_state.rendezvous_point else { + return Err(ServerError::P2P(P2PError::InvalidBehaviourConfig)); + }; + let config = RegisterConfig::new( + p2p_state.p2p_key, + p2p_state.rendezvous_point_address, + rendezvous_point, + ); + Ok(tokio::spawn(run_register_node(config, cancellation_token))) + } + } +} diff --git a/storage-provider/server/src/p2p/bootstrap.rs b/storage-provider/server/src/p2p/bootstrap.rs new file mode 100644 index 000000000..ebc8cc146 --- /dev/null +++ b/storage-provider/server/src/p2p/bootstrap.rs @@ -0,0 +1,102 @@ +use std::time::Duration; + +use libp2p::{ + futures::StreamExt, + identify, + identity::Keypair, + noise, rendezvous, + swarm::{NetworkBehaviour, SwarmEvent}, + tcp, yamux, Multiaddr, Swarm, SwarmBuilder, +}; + +use super::P2PError; + +#[derive(NetworkBehaviour)] +pub struct BootstrapBehaviour { + pub rendezvous: rendezvous::server::Behaviour, + pub identify: identify::Behaviour, +} + +pub struct BootstrapConfig { + address: Multiaddr, + keypair: Keypair, +} + +impl BootstrapConfig { + pub fn new(keypair: Keypair, address: Multiaddr) -> Self { + Self { address, keypair } + } + + pub fn create_swarm(self) -> Result<(Swarm, Multiaddr), P2PError> { + let swarm = SwarmBuilder::with_existing_identity(self.keypair) + .with_tokio() + .with_tcp( + tcp::Config::default(), + noise::Config::new, + yamux::Config::default, + ) + .map_err(|_| P2PError::InvalidTcpConfig)? + .with_behaviour(|key| BootstrapBehaviour { + // Rendezvous server behaviour for serving new peers to connecting nodes. + rendezvous: rendezvous::server::Behaviour::new( + rendezvous::server::Config::default(), + ), + // The identify behaviour is used to share the external address and the public key with connecting clients. + identify: identify::Behaviour::new(identify::Config::new( + "identify/1.0.0".to_string(), + key.public(), + )), + }) + .map_err(|_| P2PError::InvalidBehaviourConfig)? + .with_swarm_config(|cfg| cfg.with_idle_connection_timeout(Duration::from_secs(10))) + .build(); + + Ok((swarm, self.address)) + } +} + +/// Run the rendezvous point (bootstrap node). +/// Listens on the given [`Multiaddr`] +pub(crate) async fn bootstrap( + mut swarm: Swarm, + addr: Multiaddr, +) -> Result<(), P2PError> { + tracing::info!("Starting P2P bootstrap node at {addr}"); + swarm.listen_on(addr)?; + while let Some(event) = swarm.next().await { + match event { + SwarmEvent::ConnectionEstablished { peer_id, .. } => { + tracing::info!("Connected to {}", peer_id); + } + SwarmEvent::ConnectionClosed { peer_id, .. } => { + tracing::info!("Disconnected from {}", peer_id); + } + SwarmEvent::Behaviour(BootstrapBehaviourEvent::Rendezvous( + rendezvous::server::Event::PeerRegistered { peer, registration }, + )) => { + tracing::info!( + "Peer {} registered for namespace '{}' for {} seconds", + peer, + registration.namespace, + registration.ttl + ); + } + SwarmEvent::Behaviour(BootstrapBehaviourEvent::Rendezvous( + rendezvous::server::Event::DiscoverServed { + enquirer, + registrations, + }, + )) => { + if !registrations.is_empty() { + tracing::info!( + "Served peer {} with {} new registrations", + enquirer, + registrations.len() + ); + } + } + _other => {} + } + } + Ok(()) +} diff --git a/storage-provider/server/src/p2p/mod.rs b/storage-provider/server/src/p2p/mod.rs new file mode 100644 index 000000000..7b24c0811 --- /dev/null +++ b/storage-provider/server/src/p2p/mod.rs @@ -0,0 +1,168 @@ +use std::{fmt::Display, path::PathBuf, str::FromStr}; + +use bootstrap::bootstrap; +use clap::ValueEnum; +use ed25519_dalek::{pkcs8::DecodePrivateKey, SigningKey}; +use libp2p::{identity::Keypair, rendezvous::Namespace, Multiaddr, PeerId}; +use register::register; +use serde::{de, Deserialize}; +use tokio_util::{sync::CancellationToken, task::TaskTracker}; + +mod bootstrap; +mod register; + +pub(crate) use bootstrap::BootstrapConfig; +pub(crate) use register::RegisterConfig; + +const P2P_NAMESPACE: &str = "polka-storage"; + +#[derive(Default, Debug, Clone, Copy, ValueEnum, Deserialize)] +pub enum NodeType { + #[default] + Bootstrap, + Register, +} + +impl Display for NodeType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + NodeType::Bootstrap => write!(f, "bootstrap"), + NodeType::Register => write!(f, "register"), + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum P2PError { + #[error(transparent)] + Io(#[from] std::io::Error), + #[error(transparent)] + Dial(#[from] libp2p::swarm::DialError), + #[error("Invalid TCP config for swarm")] + InvalidTcpConfig, + #[error("Invalid behaviour config for swarm")] + InvalidBehaviourConfig, + #[error(transparent)] + TOMLError(#[from] toml::de::Error), + #[error("Failed to register at rendezvous point {0}")] + RegistrationFailed(PeerId), + #[error(transparent)] + P2PTransport(#[from] libp2p::TransportError), +} + +/// State struct for P2P node. +/// Holds all the information needed for spawning a node. +/// Node can be either a bootstrap or a registration node. +pub(crate) struct P2PState { + /// P2P Node type, bootstrap or registration + pub(crate) node_type: NodeType, + + /// P2P ED25519 private key + pub(crate) p2p_key: Keypair, + + /// Rendezvous point address that the registration node connects to + /// or the bootstrap node binds to. + pub(crate) rendezvous_point_address: Multiaddr, + + /// PeerID of the bootstrap node used by the registration node. + /// Optional because it is not used by the bootstrap node. + pub(crate) rendezvous_point: Option, +} + +/// Deserializes a ED25519 private key into a Keypair. +/// Can either be the private key as a string or the path of a PEM file with an @ prefixed +/// Calls `keypair_value_parser` after deserializing the source string +pub(crate) fn deser_keypair<'de, D: de::Deserializer<'de>>(d: D) -> Result { + let src: String = de::Deserialize::deserialize(d)?; + keypair_value_parser(&src).map_err(de::Error::custom) +} + +/// Parses a ED25519 private key into a Keypair. +/// Takes in a private key or the path to a PEM file, depending on the @ prefix. +pub(crate) fn keypair_value_parser(src: &str) -> Result { + let key = if let Some(stripped) = src.strip_prefix('@') { + let path = PathBuf::from_str(stripped) + .map_err(|e| e.to_string())? + .canonicalize() + .map_err(|e| e.to_string())?; + SigningKey::read_pkcs8_pem_file(path).map_err(|e| e.to_string())? + } else { + let hex_key = hex::decode(src).map_err(|e| e.to_string())?; + SigningKey::try_from(hex_key.as_slice()).map_err(|e| e.to_string())? + }; + Keypair::ed25519_from_bytes(key.to_bytes()).map_err(|e| e.to_string()) +} + +/// Parses a string to an optional Peer ID. +/// Used in the [`ConfigurationArgs`] rendezvous_point field. +pub(crate) fn string_to_peer_id_option<'de, D: de::Deserializer<'de>>( + d: D, +) -> Result, D::Error> { + let s: Option = de::Deserialize::deserialize(d)?; + match s { + Some(s) => Ok(Some(PeerId::from_str(&s).map_err(de::Error::custom)?)), + None => Ok(None), + } +} + +/// Runs a bootstrap node from the given config. +/// The `CancellationToken` is used for a graceful shutdown if the user presses ctrl+c +pub async fn run_bootstrap_node( + config: BootstrapConfig, + token: CancellationToken, +) -> Result<(), P2PError> { + tracing::info!("Starting P2P bootstrap node"); + let tracker = TaskTracker::new(); + let (swarm, addr) = config.create_swarm()?; + + tokio::select! { + res = bootstrap(swarm, addr) => { + if let Err(e) = res { + tracing::error!("Failed to start P2P node. Reason: {e}"); + return Err(e); + } + }, + _ = token.cancelled() => { + tracing::info!("P2P node has been stopped by the cancellation token..."); + }, + } + + tracker.close(); + tracker.wait().await; + + Ok(()) +} + +/// Runs a registration node from the given config. +/// The `CancellationToken` is used for a graceful shutdown if the user presses ctrl+c +pub async fn run_register_node( + config: RegisterConfig, + token: CancellationToken, +) -> Result<(), P2PError> { + tracing::info!("Starting P2P register node"); + let tracker = TaskTracker::new(); + let (swarm, rendezvous_point_address, rendezvous_point) = config.create_swarm()?; + + tokio::select! { + res = register( + swarm, + rendezvous_point, + rendezvous_point_address, + None, + Namespace::from_static(P2P_NAMESPACE), + ) => { + if let Err(e) = res { + tracing::error!("Failed to start P2P node. Reason: {e}"); + return Err(e); + } + }, + _ = token.cancelled() => { + tracing::info!("P2P node has been stopped by the cancellation token..."); + }, + } + + tracker.close(); + tracker.wait().await; + + Ok(()) +} diff --git a/storage-provider/server/src/p2p/register.rs b/storage-provider/server/src/p2p/register.rs new file mode 100644 index 000000000..c5dc986d3 --- /dev/null +++ b/storage-provider/server/src/p2p/register.rs @@ -0,0 +1,141 @@ +use std::time::Duration; + +use libp2p::{ + futures::StreamExt, + identify, + identity::Keypair, + noise, + rendezvous::{self, Namespace}, + swarm::{NetworkBehaviour, SwarmEvent}, + tcp, yamux, Multiaddr, PeerId, Swarm, SwarmBuilder, +}; + +use super::P2PError; + +#[derive(NetworkBehaviour)] +pub struct RegisterBehaviour { + pub identify: identify::Behaviour, + pub rendezvous: rendezvous::client::Behaviour, +} + +pub struct RegisterConfig { + keypair: Keypair, + rendezvous_point_address: Multiaddr, + rendezvous_point: PeerId, +} + +impl RegisterConfig { + pub fn new( + keypair: Keypair, + rendezvous_point_address: Multiaddr, + rendezvous_point: PeerId, + ) -> Self { + Self { + keypair, + rendezvous_point_address, + rendezvous_point, + } + } + + pub fn create_swarm(self) -> Result<(Swarm, Multiaddr, PeerId), P2PError> { + let swarm = SwarmBuilder::with_existing_identity(self.keypair) + .with_tokio() + .with_tcp( + tcp::Config::default(), + noise::Config::new, + yamux::Config::default, + ) + .map_err(|_| P2PError::InvalidTcpConfig)? + .with_behaviour(|key| RegisterBehaviour { + // The identify behaviour is used to receive the bootstrap node's external address and public key. + identify: identify::Behaviour::new(identify::Config::new( + "identify/1.0.0".to_string(), + key.public(), + )), + // The rendezvous client behaviour allows the bootstrap node to share peer information with us. + rendezvous: rendezvous::client::Behaviour::new(key.clone()), + }) + .map_err(|_| P2PError::InvalidBehaviourConfig)? + .with_swarm_config(|cfg| cfg.with_idle_connection_timeout(Duration::from_secs(10))) + .build(); + + Ok((swarm, self.rendezvous_point_address, self.rendezvous_point)) + } +} + +/// Register the peer with the rendezvous point. +/// The ttl is how long the peer will remain registered in seconds. +pub(crate) async fn register( + mut swarm: Swarm, + rendezvous_point: PeerId, + rendezvous_point_address: Multiaddr, + ttl: Option, + namespace: Namespace, +) -> Result<(), P2PError> { + tracing::info!("Attempting to register with rendezvous point {rendezvous_point} at {rendezvous_point_address}"); + swarm.dial(rendezvous_point_address.clone())?; + + while let Some(event) = swarm.next().await { + match event { + SwarmEvent::NewListenAddr { address, .. } => { + tracing::info!("Listening on {}", address); + } + SwarmEvent::ConnectionClosed { + peer_id, + cause: Some(error), + .. + } if peer_id == rendezvous_point => { + tracing::info!("Lost connection to rendezvous point {}", error); + } + // once `/identify` did its job, we know our external address and can register + SwarmEvent::Behaviour(RegisterBehaviourEvent::Identify( + identify::Event::Received { info, .. }, + )) => { + // Register our external address. + tracing::info!("Registering external address {}", info.observed_addr); + swarm.add_external_address(info.observed_addr); + if let Err(error) = swarm.behaviour_mut().rendezvous.register( + namespace.clone(), + rendezvous_point, + ttl, + ) { + tracing::error!("Failed to register: {error}"); + return Err(P2PError::RegistrationFailed(rendezvous_point)); + } + } + SwarmEvent::Behaviour(RegisterBehaviourEvent::Rendezvous( + rendezvous::client::Event::Registered { + namespace, + ttl, + rendezvous_node, + }, + )) => { + tracing::info!( + "Registered for namespace '{}' at rendezvous point {} for the next {} seconds", + namespace, + rendezvous_node, + ttl + ); + return Ok(()); + } + SwarmEvent::Behaviour(RegisterBehaviourEvent::Rendezvous( + rendezvous::client::Event::RegisterFailed { + rendezvous_node, + namespace, + error, + }, + )) => { + tracing::error!( + "Failed to register: rendezvous_node={}, namespace={}, error_code={:?}", + rendezvous_node, + namespace, + error + ); + return Err(P2PError::RegistrationFailed(rendezvous_node)); + } + _other => {} + } + } + + Ok(()) +} diff --git a/storage-provider/server/src/rpc.rs b/storage-provider/server/src/rpc.rs index 75760e966..ffbb1cd87 100644 --- a/storage-provider/server/src/rpc.rs +++ b/storage-provider/server/src/rpc.rs @@ -14,7 +14,7 @@ use subxt::tx::Signer; use tokio::sync::mpsc::UnboundedSender; use tokio_util::sync::CancellationToken; use tower_http::cors::{Any, CorsLayer}; -use tracing::{info, instrument}; +use tracing::instrument; use crate::{ db::DealDB, @@ -222,7 +222,7 @@ pub async fn start_rpc_server( state: RpcServerState, token: CancellationToken, ) -> Result<(), std::io::Error> { - info!("Starting RPC server at {}", state.listen_address); + tracing::info!("Starting RPC server at {}", state.listen_address); let cors = CorsLayer::new() .allow_methods([Method::POST]) @@ -238,7 +238,7 @@ pub async fn start_rpc_server( let rpc = StorageProviderRpcServer::into_rpc(state); let server_handle = server.start(rpc); - info!("RPC server started"); + tracing::info!("RPC server started"); token.cancelled_owned().await; tracing::trace!("shutdown signal received, stopping the RPC server"); diff --git a/storagext/cli/Cargo.toml b/storagext/cli/Cargo.toml index 1fffaab93..a8efa95ba 100644 --- a/storagext/cli/Cargo.toml +++ b/storagext/cli/Cargo.toml @@ -17,6 +17,7 @@ clap = { workspace = true, features = ["derive", "env"] } codec.workspace = true frame-support = { workspace = true, features = ["std"] } hex = { workspace = true, features = ["serde", "std"] } +libp2p.workspace = true primitives = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } diff --git a/storagext/cli/src/cmd/storage_provider.rs b/storagext/cli/src/cmd/storage_provider.rs index 25460bc27..a974dc08d 100644 --- a/storagext/cli/src/cmd/storage_provider.rs +++ b/storagext/cli/src/cmd/storage_provider.rs @@ -1,6 +1,7 @@ use std::time::Duration; use clap::Subcommand; +use libp2p::PeerId; use primitives::{proofs::RegisteredPoStProof, sector::SectorNumber}; use storagext::{ deser::DeserializablePath, @@ -36,8 +37,8 @@ pub enum StorageProviderCommand { /// Register account as a Storage Provider, so it can perform duties in Storage Provider Pallet. #[command(name = "register")] RegisterStorageProvider { - /// PeerId in Storage Provider P2P network, can be any String. - peer_id: String, + /// PeerId in Storage Provider P2P network. + peer_id: PeerId, /// Proof of Space Time type. /// Can only be "2KiB" meaning `RegisteredPoStProof::StackedDRGWindow2KiBV1P1`. #[arg(long, value_parser = parse_post_proof, default_value = "2KiB")] @@ -252,7 +253,7 @@ impl StorageProviderCommand { async fn register_storage_provider( client: Client, account_keypair: MultiPairSigner, - peer_id: String, + peer_id: PeerId, post_proof: RegisteredPoStProof, wait_for_finalization: bool, ) -> Result>, subxt::Error> @@ -260,12 +261,7 @@ impl StorageProviderCommand { Client: StorageProviderClientExt, { let submission_result = client - .register_storage_provider( - &account_keypair, - peer_id.clone(), - post_proof, - wait_for_finalization, - ) + .register_storage_provider(&account_keypair, peer_id, post_proof, wait_for_finalization) .await? .inspect(|result| { tracing::debug!( diff --git a/storagext/lib/Cargo.toml b/storagext/lib/Cargo.toml index 3f776749c..0357c7442 100644 --- a/storagext/lib/Cargo.toml +++ b/storagext/lib/Cargo.toml @@ -34,6 +34,7 @@ url = { workspace = true } # Optional dependencies clap = { workspace = true, optional = true, features = ["derive"] } +libp2p.workspace = true [dev-dependencies] # "Duplicated" so we can always test clap-dependent things diff --git a/storagext/lib/src/clients/storage_provider.rs b/storagext/lib/src/clients/storage_provider.rs index e2b79359c..ba32da503 100644 --- a/storagext/lib/src/clients/storage_provider.rs +++ b/storagext/lib/src/clients/storage_provider.rs @@ -1,5 +1,6 @@ use std::future::Future; +use libp2p::PeerId as P2PPeerId; use primitives::proofs::RegisteredPoStProof; use runtime::runtime_types::bounded_collections::bounded_vec::BoundedVec; use subxt::{ @@ -28,7 +29,7 @@ pub trait StorageProviderClientExt { fn register_storage_provider( &self, account_keypair: &Keypair, - peer_id: String, + peer_id: P2PPeerId, post_proof: RegisteredPoStProof, wait_for_finalization: bool, ) -> impl Future>, subxt::Error>> @@ -176,7 +177,7 @@ impl StorageProviderClientExt for crate::runtime::client::Client { async fn register_storage_provider( &self, account_keypair: &Keypair, - peer_id: String, + peer_id: P2PPeerId, post_proof: RegisteredPoStProof, wait_for_finalization: bool, ) -> Result>, subxt::Error> @@ -185,7 +186,7 @@ impl StorageProviderClientExt for crate::runtime::client::Client { { let payload = runtime::tx() .storage_provider() - .register_storage_provider(peer_id.into_bounded_byte_vec(), post_proof); + .register_storage_provider(peer_id.to_bytes().into_bounded_byte_vec(), post_proof); self.traced_submission(&payload, account_keypair, wait_for_finalization) .await