From c58ed056661ccdcd634ba9ce79717f58879ada62 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Tue, 2 Apr 2024 22:36:33 +0100 Subject: [PATCH 001/205] fix: use correct release type in upgrade process The faucet and daemon upgrades did not work correctly because the release type was hard coded to `ReleaseType::Safenode` and was not changed when this code was refactored. --- sn_node_manager/src/cmd/mod.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sn_node_manager/src/cmd/mod.rs b/sn_node_manager/src/cmd/mod.rs index 27f5436c6e..02a3e5a93f 100644 --- a/sn_node_manager/src/cmd/mod.rs +++ b/sn_node_manager/src/cmd/mod.rs @@ -47,10 +47,8 @@ pub async fn download_and_get_upgrade_bin_path( download_and_extract_release(release_type, Some(url), None, &*release_repo).await?; Ok((upgrade_bin_path, Version::parse(&version)?)) } else { - println!("Retrieving latest version of safenode..."); - let latest_version = release_repo - .get_latest_version(&ReleaseType::Safenode) - .await?; + println!("Retrieving latest version of {}...", release_type); + let latest_version = release_repo.get_latest_version(&release_type).await?; println!("Latest version is {latest_version}"); let (upgrade_bin_path, _) = download_and_extract_release( ReleaseType::Safenode, From 954e6da6e0cfaf51518661e8b2a14bd37a186930 Mon Sep 17 00:00:00 2001 From: qima Date: Wed, 3 Apr 2024 16:02:34 +0800 Subject: [PATCH 002/205] fix(node): replication_list in range filter --- sn_networking/src/replication_fetcher.rs | 42 ++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/sn_networking/src/replication_fetcher.rs b/sn_networking/src/replication_fetcher.rs index 31e69b2b3f..c3188bc5d1 100644 --- a/sn_networking/src/replication_fetcher.rs +++ b/sn_networking/src/replication_fetcher.rs @@ -97,12 +97,12 @@ impl ReplicationFetcher { let mut out_of_range_keys = vec![]; let total_incoming_keys = incoming_keys.len(); - // Filter out those out_of_range ones among the imcoming _keys. + // Filter out those out_of_range ones among the imcoming_keys. if let Some(ref distance_range) = self.distance_range { let self_address = NetworkAddress::from_peer(self.self_peer_id); incoming_keys.retain(|(addr, _record_type)| { - let is_in_range = self_address.distance(addr) >= *distance_range; + let is_in_range = self_address.distance(addr) <= *distance_range; if !is_in_range { out_of_range_keys.push(addr.clone()); } @@ -357,4 +357,42 @@ mod tests { Ok(()) } + + #[test] + fn verify_in_range_check() { + //random peer_id + let peer_id = PeerId::random(); + let self_address = NetworkAddress::from_peer(peer_id); + let (event_sender, _event_receiver) = mpsc::channel(4); + let mut replication_fetcher = ReplicationFetcher::new(peer_id, event_sender); + + // Set distance range + let distance_target = NetworkAddress::from_peer(PeerId::random()); + let distance_range = self_address.distance(&distance_target); + replication_fetcher.set_distance_range(distance_range); + + let mut incoming_keys = Vec::new(); + let mut in_range_keys = 0; + (0..100).for_each(|_| { + let random_data: Vec = (0..50).map(|_| rand::random::()).collect(); + let key = NetworkAddress::from_record_key(&RecordKey::from(random_data)); + + if key.distance(&self_address) <= distance_range { + in_range_keys += 1; + } + + incoming_keys.push((key, RecordType::Chunk)); + }); + + let keys_to_fetch = + replication_fetcher.add_keys(PeerId::random(), incoming_keys, &Default::default()); + assert_eq!( + keys_to_fetch.len(), + replication_fetcher.on_going_fetches.len() + ); + assert_eq!( + in_range_keys, + keys_to_fetch.len() + replication_fetcher.to_be_fetched.len() + ); + } } From 568e68a13d3f4bf7eac569109e7b32ec41f93e5b Mon Sep 17 00:00:00 2001 From: qima Date: Wed, 3 Apr 2024 18:18:04 +0800 Subject: [PATCH 003/205] fix(node): replication_fetch keep distance_range sync with record_store --- sn_networking/src/driver.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index 22d060b132..90fba56a97 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -624,6 +624,8 @@ impl SwarmDriver { if let Some(distance) = self.get_farthest_relevant_address_estimate(&closest_k_peers) { // set any new distance to fathest record in the store self.swarm.behaviour_mut().kademlia.store_mut().set_distance_range(distance); + // the distance range within the replication_fetcher shall be in sync as well + self.replication_fetcher.set_distance_range(distance); } } } From 10acc176e7d8790d7f7bd3b7834008c73a33d915 Mon Sep 17 00:00:00 2001 From: qima Date: Wed, 3 Apr 2024 19:20:34 +0800 Subject: [PATCH 004/205] chore(node): extend distance range --- sn_networking/src/driver.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index 90fba56a97..e6379738c9 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -649,11 +649,11 @@ impl SwarmDriver { let our_address = NetworkAddress::from_peer(self.self_peer_id); - // get REPLICATE_RANGE + 2 peer's address distance + // get REPLICATE_RANGE + 2 peer's address distance // This is a rough estimate of the farthest address we might be responsible for. // We want this to be higher than actually necessary, so we retain more data // and can be sure to pass bad node checks - if let Some(peer) = closest_k_peers.get(REPLICATE_RANGE) { + if let Some(peer) = closest_k_peers.get(REPLICATE_RANGE + 2) { let address = NetworkAddress::from_peer(*peer); let distance = our_address.distance(&address); farthest_distance = Some(distance); From cef8ecf5955dd6690414230acc36ccecd1b0b59c Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 2 Apr 2024 22:17:52 +0530 Subject: [PATCH 005/205] feat(network): provide network versioning --- Cargo.lock | 2 + sn_build_info/src/lib.rs | 8 ++-- sn_networking/Cargo.toml | 2 + sn_networking/src/driver.rs | 45 +++++++------------- sn_networking/src/event.rs | 12 +++--- sn_networking/src/lib.rs | 1 + sn_networking/src/version.rs | 81 ++++++++++++++++++++++++++++++++++++ 7 files changed, 110 insertions(+), 41 deletions(-) create mode 100644 sn_networking/src/version.rs diff --git a/Cargo.lock b/Cargo.lock index 06853bb39b..a1702bd5eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5253,6 +5253,7 @@ dependencies = [ "hex", "hyper", "itertools 0.11.0", + "lazy_static", "libp2p", "libp2p-identity", "prometheus-client", @@ -5261,6 +5262,7 @@ dependencies = [ "rayon", "rmp-serde", "serde", + "sn_build_info", "sn_protocol", "sn_registers", "sn_transfers", diff --git a/sn_build_info/src/lib.rs b/sn_build_info/src/lib.rs index 1eeb573efe..82e690eb5a 100644 --- a/sn_build_info/src/lib.rs +++ b/sn_build_info/src/lib.rs @@ -7,7 +7,7 @@ // permissions and limitations relating to use of the SAFE Network Software. /// Git information separated by slashes: ` / / ` -pub fn git_info() -> &'static str { +pub const fn git_info() -> &'static str { concat!( env!("VERGEN_GIT_SHA"), " / ", @@ -18,16 +18,16 @@ pub fn git_info() -> &'static str { } /// Annotated tag description, or fall back to abbreviated commit object. -pub fn git_describe() -> &'static str { +pub const fn git_describe() -> &'static str { env!("VERGEN_GIT_DESCRIBE") } /// Current git branch. -pub fn git_branch() -> &'static str { +pub const fn git_branch() -> &'static str { env!("VERGEN_GIT_BRANCH") } /// Shortened SHA-1 hash. -pub fn git_sha() -> &'static str { +pub const fn git_sha() -> &'static str { env!("VERGEN_GIT_SHA") } diff --git a/sn_networking/Cargo.toml b/sn_networking/Cargo.toml index 270f0f7999..7e4f67d947 100644 --- a/sn_networking/Cargo.toml +++ b/sn_networking/Cargo.toml @@ -31,6 +31,7 @@ hyper = { version = "0.14", features = [ ], optional = true } itertools = "~0.11.0" custom_debug = "~0.5.0" +lazy_static = "~1.4.0" libp2p = { version = "0.53", features = [ "tokio", "dns", @@ -49,6 +50,7 @@ rand = { version = "~0.8.5", features = ["small_rng"] } rayon = "1.8.0" rmp-serde = "1.1.1" serde = { version = "1.0.133", features = ["derive", "rc"] } +sn_build_info = { path="../sn_build_info", version = "0.1.5" } sn_protocol = { path = "../sn_protocol", version = "0.16.1" } sn_transfers = { path = "../sn_transfers", version = "0.17.1" } sn_registers = { path = "../sn_registers", version = "0.3.12" } diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index e6379738c9..1af6bc3b2a 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -11,6 +11,10 @@ use crate::metrics::NetworkMetrics; #[cfg(feature = "open-metrics")] use crate::metrics_service::run_metrics_server; use crate::target_arch::{interval, spawn, Instant}; +use crate::version::{ + IDENTIFY_CLIENT_VERSION_STR, IDENTIFY_NODE_VERSION_STR, IDENTIFY_PROTOCOL_STR, + REQ_RESPONSE_VERSION_STR, +}; use crate::{ bootstrap::{ContinuousBootstrap, BOOTSTRAP_INTERVAL}, circular_vec::CircularVec, @@ -87,33 +91,11 @@ const REQUEST_TIMEOUT_DEFAULT_S: Duration = Duration::from_secs(30); // Sets the keep-alive timeout of idle connections. const CONNECTION_KEEP_ALIVE_TIMEOUT: Duration = Duration::from_secs(30); -/// The suffix is the version of the node. -const SN_NODE_VERSION_STR: &str = concat!("safe/node/", env!("CARGO_PKG_VERSION")); -/// / first version for the req/response protocol -const REQ_RESPONSE_VERSION_STR: &str = concat!("/safe/node/", env!("CARGO_PKG_VERSION")); - -/// The suffix is the version of the client. -const IDENTIFY_CLIENT_VERSION_STR: &str = concat!("safe/client/", env!("CARGO_PKG_VERSION")); -const IDENTIFY_PROTOCOL_STR: &str = concat!("safe/", env!("CARGO_PKG_VERSION")); - const NETWORKING_CHANNEL_SIZE: usize = 10_000; /// Time before a Kad query times out if no response is received const KAD_QUERY_TIMEOUT_S: Duration = Duration::from_secs(10); -// Protocol support shall be downward compatible for patch only version update. -// i.e. versions of `A.B.X` shall be considered as a same protocol of `A.B` -pub(crate) fn truncate_patch_version(full_str: &str) -> &str { - if full_str.matches('.').count() == 2 { - match full_str.rfind('.') { - Some(pos) => &full_str[..pos], - None => full_str, - } - } else { - full_str - } -} - /// The various settings to apply to when fetching a record from network #[derive(Clone)] pub struct GetRecordCfg { @@ -309,7 +291,7 @@ impl NetworkBuilder { Some(store_cfg), false, ProtocolSupport::Full, - truncate_patch_version(SN_NODE_VERSION_STR).to_string(), + IDENTIFY_NODE_VERSION_STR.to_string(), )?; // Listen on the provided address @@ -361,7 +343,7 @@ impl NetworkBuilder { None, true, ProtocolSupport::Outbound, - truncate_patch_version(IDENTIFY_CLIENT_VERSION_STR).to_string(), + IDENTIFY_CLIENT_VERSION_STR.to_string(), )?; Ok((network, net_event_recv, driver)) @@ -401,9 +383,13 @@ impl NetworkBuilder { let cfg = RequestResponseConfig::default() .with_request_timeout(self.request_timeout.unwrap_or(REQUEST_TIMEOUT_DEFAULT_S)); + info!( + "Building request response with {:?}", + REQ_RESPONSE_VERSION_STR.as_str() + ); request_response::cbor::Behaviour::new( [( - StreamProtocol::new(truncate_patch_version(REQ_RESPONSE_VERSION_STR)), + StreamProtocol::new(&REQ_RESPONSE_VERSION_STR), req_res_protocol, )], cfg, @@ -452,12 +438,11 @@ impl NetworkBuilder { let mdns = mdns::tokio::Behaviour::new(mdns_config, peer_id)?; // Identify Behaviour + let identify_protocol_str = IDENTIFY_PROTOCOL_STR.to_string(); + info!("Building Identify with identify_protocol_str: {identify_protocol_str:?} and identify_version: {identify_version:?}"); let identify = { - let cfg = libp2p::identify::Config::new( - truncate_patch_version(IDENTIFY_PROTOCOL_STR).to_string(), - self.keypair.public(), - ) - .with_agent_version(identify_version); + let cfg = libp2p::identify::Config::new(identify_protocol_str, self.keypair.public()) + .with_agent_version(identify_version); libp2p::identify::Behaviour::new(cfg) }; diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 9347624052..2c440fa450 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -8,10 +8,11 @@ use crate::{ cmd::SwarmCmd, - driver::{truncate_patch_version, PendingGetClosestType, SwarmDriver}, + driver::{PendingGetClosestType, SwarmDriver}, error::{NetworkError, Result}, - multiaddr_is_global, multiaddr_strip_p2p, sort_peers_by_address, CLOSE_GROUP_SIZE, - REPLICATE_RANGE, + multiaddr_is_global, multiaddr_strip_p2p, sort_peers_by_address, + version::IDENTIFY_NODE_VERSION_STR, + CLOSE_GROUP_SIZE, REPLICATE_RANGE, }; use core::fmt; use custom_debug::Debug as CustomDebug; @@ -47,9 +48,6 @@ use tokio::sync::oneshot; use tokio::time::Duration; use tracing::{info, warn}; -/// Our agent string has as a prefix that we can match against. -const IDENTIFY_AGENT_STR: &str = concat!("safe/node/", env!("CARGO_PKG_VERSION")); - /// NodeEvent enum #[derive(CustomDebug)] pub(super) enum NodeEvent { @@ -226,7 +224,7 @@ impl SwarmDriver { let has_dialed = self.dialed_peers.contains(&peer_id); let peer_is_agent = info .agent_version - .starts_with(truncate_patch_version(IDENTIFY_AGENT_STR)); + .starts_with(&IDENTIFY_NODE_VERSION_STR.to_string()); // If we're not in local mode, only add globally reachable addresses. // Strip the `/p2p/...` part of the multiaddresses. diff --git a/sn_networking/src/lib.rs b/sn_networking/src/lib.rs index 31a6d4523f..4a17fc9525 100644 --- a/sn_networking/src/lib.rs +++ b/sn_networking/src/lib.rs @@ -29,6 +29,7 @@ mod spends; pub mod target_arch; mod transfers; mod transport; +mod version; // re-export arch dependent deps for use in the crate, or above pub use target_arch::{interval, sleep, spawn, Instant, Interval}; diff --git a/sn_networking/src/version.rs b/sn_networking/src/version.rs new file mode 100644 index 0000000000..527b51a3bf --- /dev/null +++ b/sn_networking/src/version.rs @@ -0,0 +1,81 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use lazy_static::lazy_static; + +/// Set this env variable to provide custom network versioning. If it is set to 'restricted', then the git branch name +/// is used as the version string. Else we directly use the passed in string as the version. +const NETWORK_VERSION_MODE_ENV_VARIABLE: &str = "NETWORK_VERSION_MODE"; + +lazy_static! { + /// The node version used during Identify Behaviour. + pub static ref IDENTIFY_NODE_VERSION_STR: String = + format!( + "safe{}/node/{}", + get_network_version(), + get_truncate_version_str() + ); + + /// The client version used during Identify Behaviour. + pub static ref IDENTIFY_CLIENT_VERSION_STR: String = + format!( + "safe{}/client/{}", + get_network_version(), + get_truncate_version_str() + ); + + /// / first version for the req/response protocol + pub static ref REQ_RESPONSE_VERSION_STR: String = + format!( + "/safe{}/node/{}", + get_network_version(), + get_truncate_version_str() + ); + + + /// The identify protocol version + pub static ref IDENTIFY_PROTOCOL_STR: String = + format!( + "safe{}/{}", + get_network_version(), + get_truncate_version_str() + ); + + +} + +// Protocol support shall be downward compatible for patch only version update. +// i.e. versions of `A.B.X` shall be considered as a same protocol of `A.B` +fn get_truncate_version_str() -> &'static str { + let version_str = env!("CARGO_PKG_VERSION"); + if version_str.matches('.').count() == 2 { + match version_str.rfind('.') { + Some(pos) => &version_str[..pos], + None => version_str, + } + } else { + version_str + } +} + +/// Get the network version string. +/// If the network version mode env variable is set to `restricted`, then the git branch is used as the version. +/// Else any non empty string is used as the version string. +/// If the env variable is empty or not set, then we do not apply any network versioning. +fn get_network_version() -> String { + match std::env::var(NETWORK_VERSION_MODE_ENV_VARIABLE) { + Ok(value) if !value.is_empty() => { + if value == "restricted" { + format!("/{}", sn_build_info::git_branch()) + } else { + format!("/{value}") + } + } + _ => "".to_string(), + } +} From 4c6ee0b62517c5c97cc9203deee81fc0864cc71e Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 3 Apr 2024 15:39:46 +0530 Subject: [PATCH 006/205] feat(release): set network versioning based on the branch --- .github/workflows/release.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5856131b6c..7579ff7f4c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,6 +8,7 @@ on: push: branches: - main + - stable - alpha* - beta* - rc* @@ -124,9 +125,23 @@ jobs: rm just-1.13.0-x86_64-unknown-linux-musl.tar.gz sudo mv just/just /usr/local/bin rm -rf just + + # Set the network versioning based on our branch + # main/stable branches defaults to open compatibility. Other branches will be versioned by their branch name. + - name: provide network versioning + shell: bash + run: | + if [[ "$GITHUB_REF_NAME" != "main" && "$GITHUB_REF_NAME" != "stable" ]]; then + echo "NETWORK_VERSION_MODE=restricted" >> $GITHUB_ENV + fi + - name: publish and release shell: bash run: | + # make sure the version env variable is set + export NETWORK_VERSION_MODE=${{ env.NETWORK_VERSION_MODE }} + echo "NETWORK_VERSION_MODE=$NETWORK_VERSION_MODE" + # Package versioned assets as tar.gz and zip archives, and upload them to S3. # # This is done before publishing because the node manager relies on these binaries @@ -137,6 +152,7 @@ jobs: # because the process gets the latest version from `crates.io` then downloads the binaries # from S3, using that version number. Uploading the binaries to S3 before publishing # ensures that they will exist after the new crate has been published. + just package-release-assets "safe" just package-release-assets "safenode" just package-release-assets "faucet" @@ -171,6 +187,7 @@ jobs: just upload-release-assets-to-s3 "safenodemand" just upload-release-assets-to-s3 "faucet" just upload-release-assets-to-s3 "safenode_rpc_client" + - name: post notification to slack on failure if: ${{ failure() }} uses: bryannice/gitactions-slack-notification@2.0.0 From a15b58b9b15c622065dff6320029d5fdd487dbd6 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 3 Apr 2024 17:49:32 +0530 Subject: [PATCH 007/205] feat(network): network contacts url should point to the correct network version --- Cargo.lock | 6 ++- sn_networking/Cargo.toml | 2 - sn_networking/src/driver.rs | 11 ++-- sn_networking/src/event.rs | 6 +-- sn_networking/src/lib.rs | 1 - sn_peers_acquisition/Cargo.toml | 2 + sn_peers_acquisition/src/lib.rs | 16 ++++-- sn_protocol/Cargo.toml | 2 + sn_protocol/src/lib.rs | 2 + {sn_networking => sn_protocol}/src/version.rs | 52 +++++++++++-------- 10 files changed, 61 insertions(+), 39 deletions(-) rename {sn_networking => sn_protocol}/src/version.rs (82%) diff --git a/Cargo.lock b/Cargo.lock index a1702bd5eb..cbf461ad44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5253,7 +5253,6 @@ dependencies = [ "hex", "hyper", "itertools 0.11.0", - "lazy_static", "libp2p", "libp2p-identity", "prometheus-client", @@ -5262,7 +5261,6 @@ dependencies = [ "rayon", "rmp-serde", "serde", - "sn_build_info", "sn_protocol", "sn_registers", "sn_transfers", @@ -5366,9 +5364,11 @@ name = "sn_peers_acquisition" version = "0.2.8" dependencies = [ "clap 4.5.4", + "lazy_static", "libp2p", "rand", "reqwest", + "sn_protocol", "thiserror", "tokio", "tracing", @@ -5386,12 +5386,14 @@ dependencies = [ "custom_debug", "dirs-next", "hex", + "lazy_static", "libp2p", "prost 0.9.0", "rmp-serde", "serde", "serde_json", "sha2", + "sn_build_info", "sn_registers", "sn_transfers", "thiserror", diff --git a/sn_networking/Cargo.toml b/sn_networking/Cargo.toml index 7e4f67d947..270f0f7999 100644 --- a/sn_networking/Cargo.toml +++ b/sn_networking/Cargo.toml @@ -31,7 +31,6 @@ hyper = { version = "0.14", features = [ ], optional = true } itertools = "~0.11.0" custom_debug = "~0.5.0" -lazy_static = "~1.4.0" libp2p = { version = "0.53", features = [ "tokio", "dns", @@ -50,7 +49,6 @@ rand = { version = "~0.8.5", features = ["small_rng"] } rayon = "1.8.0" rmp-serde = "1.1.1" serde = { version = "1.0.133", features = ["derive", "rc"] } -sn_build_info = { path="../sn_build_info", version = "0.1.5" } sn_protocol = { path = "../sn_protocol", version = "0.16.1" } sn_transfers = { path = "../sn_transfers", version = "0.17.1" } sn_registers = { path = "../sn_registers", version = "0.3.12" } diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index 1af6bc3b2a..c9f0148b95 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -10,11 +10,6 @@ use crate::metrics::NetworkMetrics; #[cfg(feature = "open-metrics")] use crate::metrics_service::run_metrics_server; -use crate::target_arch::{interval, spawn, Instant}; -use crate::version::{ - IDENTIFY_CLIENT_VERSION_STR, IDENTIFY_NODE_VERSION_STR, IDENTIFY_PROTOCOL_STR, - REQ_RESPONSE_VERSION_STR, -}; use crate::{ bootstrap::{ContinuousBootstrap, BOOTSTRAP_INTERVAL}, circular_vec::CircularVec, @@ -28,6 +23,7 @@ use crate::{ record_store::{ClientRecordStore, NodeRecordStore, NodeRecordStoreConfig}, record_store_api::UnifiedRecordStore, replication_fetcher::ReplicationFetcher, + target_arch::{interval, spawn, Instant}, transport, Network, CLOSE_GROUP_SIZE, }; use crate::{NodeIssue, REPLICATE_RANGE}; @@ -35,7 +31,6 @@ use futures::StreamExt; use libp2p::kad::KBucketDistance as Distance; #[cfg(feature = "local-discovery")] use libp2p::mdns; - use libp2p::{ identity::Keypair, kad::{self, QueryId, Quorum, Record, K_VALUE}, @@ -52,6 +47,10 @@ use prometheus_client::registry::Registry; use sn_protocol::{ messages::{ChunkProof, Nonce, Request, Response}, storage::RetryStrategy, + version::{ + IDENTIFY_CLIENT_VERSION_STR, IDENTIFY_NODE_VERSION_STR, IDENTIFY_PROTOCOL_STR, + REQ_RESPONSE_VERSION_STR, + }, NetworkAddress, PrettyPrintKBucketKey, PrettyPrintRecordKey, }; use sn_transfers::PaymentQuote; diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 2c440fa450..28add13d5a 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -11,7 +11,7 @@ use crate::{ driver::{PendingGetClosestType, SwarmDriver}, error::{NetworkError, Result}, multiaddr_is_global, multiaddr_strip_p2p, sort_peers_by_address, - version::IDENTIFY_NODE_VERSION_STR, + target_arch::Instant, CLOSE_GROUP_SIZE, REPLICATE_RANGE, }; use core::fmt; @@ -29,10 +29,8 @@ use libp2p::{ }, Multiaddr, PeerId, TransportError, }; - -use crate::target_arch::Instant; - use rand::{rngs::OsRng, Rng}; +use sn_protocol::version::IDENTIFY_NODE_VERSION_STR; use sn_protocol::{ get_port_from_multiaddr, messages::{CmdResponse, Query, Request, Response}, diff --git a/sn_networking/src/lib.rs b/sn_networking/src/lib.rs index 4a17fc9525..31a6d4523f 100644 --- a/sn_networking/src/lib.rs +++ b/sn_networking/src/lib.rs @@ -29,7 +29,6 @@ mod spends; pub mod target_arch; mod transfers; mod transport; -mod version; // re-export arch dependent deps for use in the crate, or above pub use target_arch::{interval, sleep, spawn, Instant, Interval}; diff --git a/sn_peers_acquisition/Cargo.toml b/sn_peers_acquisition/Cargo.toml index fdc652334f..d0ed166892 100644 --- a/sn_peers_acquisition/Cargo.toml +++ b/sn_peers_acquisition/Cargo.toml @@ -17,9 +17,11 @@ websockets = [] [dependencies] clap = { version = "4.2.1", features = ["derive", "env"] } +lazy_static = "~1.4.0" libp2p = { version="0.53", features = [] } rand = "0.8.5" reqwest = { version="0.11.18", default-features=false, features = ["rustls-tls"], optional = true } +sn_protocol = { path = "../sn_protocol", version = "0.16.1" } thiserror = "1.0.23" tokio = { version = "1.32.0", optional = true, default-features = false} tracing = { version = "~0.1.26" } diff --git a/sn_peers_acquisition/src/lib.rs b/sn_peers_acquisition/src/lib.rs index 8c0096ea3f..26e192a18c 100644 --- a/sn_peers_acquisition/src/lib.rs +++ b/sn_peers_acquisition/src/lib.rs @@ -10,15 +10,25 @@ pub mod error; use crate::error::{Error, Result}; use clap::Args; +#[cfg(feature = "network-contacts")] +use lazy_static::lazy_static; use libp2p::{multiaddr::Protocol, Multiaddr}; use rand::{seq::SliceRandom, thread_rng}; +#[cfg(feature = "network-contacts")] +use sn_protocol::version::get_network_version; use tracing::*; #[cfg(feature = "network-contacts")] use url::Url; #[cfg(feature = "network-contacts")] -// URL containing the multi-addresses of the bootstrap nodes. -const NETWORK_CONTACTS_URL: &str = "https://sn-testnet.s3.eu-west-2.amazonaws.com/network-contacts"; +lazy_static! { + // URL containing the multi-addresses of the bootstrap nodes. + pub static ref NETWORK_CONTACTS_URL: String = { + let version = get_network_version(); + let version_prefix = if !version.is_empty() { format!("{version}-") } else { version }; + format!("https://sn-testnet.s3.eu-west-2.amazonaws.com/{version_prefix}network-contacts") + }; +} #[cfg(feature = "network-contacts")] // The maximum number of retries to be performed while trying to fetch the network contacts file. @@ -113,7 +123,7 @@ async fn get_network_contacts(args: &PeersArgs) -> Result> { let url = args .network_contacts_url .clone() - .unwrap_or(Url::parse(NETWORK_CONTACTS_URL)?); + .unwrap_or(Url::parse(NETWORK_CONTACTS_URL.as_str())?); info!("Trying to fetch the bootstrap peers from {url}"); println!("Trying to fetch the bootstrap peers from {url}"); diff --git a/sn_protocol/Cargo.toml b/sn_protocol/Cargo.toml index 71f71ae90a..81f7e3af11 100644 --- a/sn_protocol/Cargo.toml +++ b/sn_protocol/Cargo.toml @@ -22,11 +22,13 @@ crdts = { version = "7.3", default-features = false, features = ["merkle"] } custom_debug = "~0.5.0" dirs-next = "~2.0.0" hex = "~0.4.3" +lazy_static = "~1.4.0" libp2p = { version="0.53", features = ["identify", "kad"] } rmp-serde = "1.1.1" serde = { version = "1.0.133", features = [ "derive", "rc" ]} serde_json = "1.0" sha2 = "0.10.7" +sn_build_info = { path="../sn_build_info", version = "0.1.5" } sn_transfers = { path = "../sn_transfers", version = "0.17.1" } sn_registers = { path = "../sn_registers", version = "0.3.12" } thiserror = "1.0.23" diff --git a/sn_protocol/src/lib.rs b/sn_protocol/src/lib.rs index f61113e26d..fc3657c825 100644 --- a/sn_protocol/src/lib.rs +++ b/sn_protocol/src/lib.rs @@ -19,6 +19,8 @@ pub mod node; pub mod node_rpc; /// Storage types for spends, chunks and registers. pub mod storage; +/// The network versioning logic +pub mod version; // this includes code generated from .proto files #[allow(clippy::unwrap_used)] diff --git a/sn_networking/src/version.rs b/sn_protocol/src/version.rs similarity index 82% rename from sn_networking/src/version.rs rename to sn_protocol/src/version.rs index 527b51a3bf..1c5a6bac24 100644 --- a/sn_networking/src/version.rs +++ b/sn_protocol/src/version.rs @@ -17,7 +17,7 @@ lazy_static! { pub static ref IDENTIFY_NODE_VERSION_STR: String = format!( "safe{}/node/{}", - get_network_version(), + write_network_version_with_slash(), get_truncate_version_str() ); @@ -25,7 +25,7 @@ lazy_static! { pub static ref IDENTIFY_CLIENT_VERSION_STR: String = format!( "safe{}/client/{}", - get_network_version(), + write_network_version_with_slash(), get_truncate_version_str() ); @@ -33,7 +33,7 @@ lazy_static! { pub static ref REQ_RESPONSE_VERSION_STR: String = format!( "/safe{}/node/{}", - get_network_version(), + write_network_version_with_slash(), get_truncate_version_str() ); @@ -42,40 +42,50 @@ lazy_static! { pub static ref IDENTIFY_PROTOCOL_STR: String = format!( "safe{}/{}", - get_network_version(), + write_network_version_with_slash(), get_truncate_version_str() ); } -// Protocol support shall be downward compatible for patch only version update. -// i.e. versions of `A.B.X` shall be considered as a same protocol of `A.B` -fn get_truncate_version_str() -> &'static str { - let version_str = env!("CARGO_PKG_VERSION"); - if version_str.matches('.').count() == 2 { - match version_str.rfind('.') { - Some(pos) => &version_str[..pos], - None => version_str, - } - } else { - version_str - } -} - /// Get the network version string. /// If the network version mode env variable is set to `restricted`, then the git branch is used as the version. /// Else any non empty string is used as the version string. /// If the env variable is empty or not set, then we do not apply any network versioning. -fn get_network_version() -> String { +pub fn get_network_version() -> String { match std::env::var(NETWORK_VERSION_MODE_ENV_VARIABLE) { Ok(value) if !value.is_empty() => { if value == "restricted" { - format!("/{}", sn_build_info::git_branch()) + sn_build_info::git_branch().to_string() } else { - format!("/{value}") + value } } _ => "".to_string(), } } + +/// Helper to write the network version with `/` appended if it is not empty +fn write_network_version_with_slash() -> String { + let version = get_network_version(); + if version.is_empty() { + version + } else { + format!("/{version}") + } +} + +// Protocol support shall be downward compatible for patch only version update. +// i.e. versions of `A.B.X` shall be considered as a same protocol of `A.B` +fn get_truncate_version_str() -> &'static str { + let version_str = env!("CARGO_PKG_VERSION"); + if version_str.matches('.').count() == 2 { + match version_str.rfind('.') { + Some(pos) => &version_str[..pos], + None => version_str, + } + } else { + version_str + } +} From 1776619c0ee33fa5ca841f5194ecc1112125ca60 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 3 Apr 2024 19:00:49 +0530 Subject: [PATCH 008/205] feat(release): set network versioning during build process not the release --- .github/workflows/release.yml | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7579ff7f4c..46bd8631b2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,8 +46,24 @@ jobs: # It's quite slow to install just by building it, but here we need a cross-platform solution. - shell: bash run: cargo install just - - shell: bash - run: just build-release-artifacts "${{ matrix.target }}" + + # Set the network versioning based on our branch + # main/stable branches defaults to open compatibility. Other branches will be versioned by their branch name. + - name: provide network versioning + shell: bash + run: | + if [[ "$GITHUB_REF_NAME" != "main" && "$GITHUB_REF_NAME" != "stable" ]]; then + echo "NETWORK_VERSION_MODE=restricted" >> $GITHUB_ENV + fi + + - name: build release artifacts + shell: bash + run: | + # make sure the version env variable is set + export NETWORK_VERSION_MODE=${{ env.NETWORK_VERSION_MODE }} + echo "NETWORK_VERSION_MODE=$NETWORK_VERSION_MODE" + just build-release-artifacts "${{ matrix.target }}" + - uses: actions/upload-artifact@main with: name: safe_network-${{ matrix.target }} @@ -126,22 +142,9 @@ jobs: sudo mv just/just /usr/local/bin rm -rf just - # Set the network versioning based on our branch - # main/stable branches defaults to open compatibility. Other branches will be versioned by their branch name. - - name: provide network versioning - shell: bash - run: | - if [[ "$GITHUB_REF_NAME" != "main" && "$GITHUB_REF_NAME" != "stable" ]]; then - echo "NETWORK_VERSION_MODE=restricted" >> $GITHUB_ENV - fi - - name: publish and release shell: bash run: | - # make sure the version env variable is set - export NETWORK_VERSION_MODE=${{ env.NETWORK_VERSION_MODE }} - echo "NETWORK_VERSION_MODE=$NETWORK_VERSION_MODE" - # Package versioned assets as tar.gz and zip archives, and upload them to S3. # # This is done before publishing because the node manager relies on these binaries From b662acff0fbcf0801364e8bfb0b649b360c88945 Mon Sep 17 00:00:00 2001 From: qima Date: Thu, 4 Apr 2024 13:50:56 +0800 Subject: [PATCH 009/205] chore(node): pass entire QuotingMetrics into calculate_cost_for_records --- sn_networking/src/record_store.rs | 107 ++++++++++++++++++++---------- sn_node/src/quote.rs | 8 +-- 2 files changed, 74 insertions(+), 41 deletions(-) diff --git a/sn_networking/src/record_store.rs b/sn_networking/src/record_store.rs index e05f27a24d..c07579d1e7 100644 --- a/sn_networking/src/record_store.rs +++ b/sn_networking/src/record_store.rs @@ -450,13 +450,7 @@ impl NodeRecordStore { let cost = if self.contains(key) { 0 } else { - // vdash metric (if modified please notify at https://github.com/happybeing/vdash/issues): - calculate_cost_for_records( - quoting_metrics.close_records_stored, - quoting_metrics.received_payment_count, - quoting_metrics.max_records, - quoting_metrics.live_time, - ) + calculate_cost_for_records("ing_metrics) }; // vdash metric (if modified please notify at https://github.com/happybeing/vdash/issues): info!("Cost is now {cost:?} for quoting_metrics {quoting_metrics:?}"); @@ -698,14 +692,14 @@ impl RecordStore for ClientRecordStore { // to allow nodes receiving too many replication copies can still got paid, // and gives an exponential pricing curve when storage reaches high. // and give extra reward (lower the quoting price to gain a better chance) to long lived nodes. -pub fn calculate_cost_for_records( - records_stored: usize, - received_payment_count: usize, - max_records: usize, - live_time: u64, -) -> u64 { +pub fn calculate_cost_for_records(quoting_metrics: &QuotingMetrics) -> u64 { use std::cmp::{max, min}; + let records_stored = quoting_metrics.close_records_stored; + let received_payment_count = quoting_metrics.received_payment_count; + let max_records = quoting_metrics.max_records; + let live_time = quoting_metrics.live_time; + let ori_cost = (10 * records_stored) as u64; let divider = max(1, records_stored / max(1, received_payment_count)) as u64; @@ -783,21 +777,36 @@ mod tests { #[test] fn test_calculate_max_cost_for_records() { - let sut = calculate_cost_for_records(2049, 2049, 2048, 1); + let sut = calculate_cost_for_records(&QuotingMetrics { + close_records_stored: 2049, + max_records: 2048, + received_payment_count: 2049, + live_time: 1, + }); assert_eq!(sut, TOTAL_SUPPLY / CLOSE_GROUP_SIZE as u64); } #[test] fn test_calculate_50_percent_cost_for_records() { let percent = MAX_RECORDS_COUNT * 50 / 100; - let sut = calculate_cost_for_records(percent, percent, 2048, 1); + let sut = calculate_cost_for_records(&QuotingMetrics { + close_records_stored: percent, + max_records: 2048, + received_payment_count: percent, + live_time: 1, + }); // at this point we should be at max cost assert_eq!(sut, 10240); } #[test] fn test_calculate_60_percent_cost_for_records() { let percent = MAX_RECORDS_COUNT * 60 / 100; - let sut = calculate_cost_for_records(percent, percent, 2048, 1); + let sut = calculate_cost_for_records(&QuotingMetrics { + close_records_stored: percent, + max_records: 2048, + received_payment_count: percent, + live_time: 1, + }); // at this point we should be at max cost assert_eq!(sut, 12280); } @@ -805,7 +814,12 @@ mod tests { #[test] fn test_calculate_65_percent_cost_for_records() { let percent = MAX_RECORDS_COUNT * 65 / 100; - let sut = calculate_cost_for_records(percent, percent, 2048, 1); + let sut = calculate_cost_for_records(&QuotingMetrics { + close_records_stored: percent, + max_records: 2048, + received_payment_count: percent, + live_time: 1, + }); // at this point we should be at max cost assert_eq!(sut, 2023120); } @@ -813,7 +827,12 @@ mod tests { #[test] fn test_calculate_70_percent_cost_for_records() { let percent = MAX_RECORDS_COUNT * 70 / 100; - let sut = calculate_cost_for_records(percent, percent, 2048, 1); + let sut = calculate_cost_for_records(&QuotingMetrics { + close_records_stored: percent, + max_records: 2048, + received_payment_count: percent, + live_time: 1, + }); // at this point we should be at max cost assert_eq!(sut, 316248770); } @@ -821,7 +840,12 @@ mod tests { #[test] fn test_calculate_80_percent_cost_for_records() { let percent = MAX_RECORDS_COUNT * 80 / 100; - let sut = calculate_cost_for_records(percent, percent, 2048, 1); + let sut = calculate_cost_for_records(&QuotingMetrics { + close_records_stored: percent, + max_records: 2048, + received_payment_count: percent, + live_time: 1, + }); // at this point we should be at max cost assert_eq!(sut, 7978447975680); } @@ -829,14 +853,24 @@ mod tests { #[test] fn test_calculate_90_percent_cost_for_records() { let percent = MAX_RECORDS_COUNT * 90 / 100; - let sut = calculate_cost_for_records(percent, percent, 2048, 1); + let sut = calculate_cost_for_records(&QuotingMetrics { + close_records_stored: percent, + max_records: 2048, + received_payment_count: percent, + live_time: 1, + }); // at this point we should be at max cost assert_eq!(sut, 198121748221132800); } #[test] fn test_calculate_min_cost_for_records() { - let sut = calculate_cost_for_records(0, 0, 2048, 1); + let sut = calculate_cost_for_records(&QuotingMetrics { + close_records_stored: 0, + max_records: 2048, + received_payment_count: 0, + live_time: 1, + }); assert_eq!(sut, 10); } @@ -1168,12 +1202,12 @@ mod tests { let (close_records_stored, nanos_earnt, received_payment_count) = peers.entry(*peer).or_insert((0, 0, 0)); if *peer == payee { - let cost = calculate_cost_for_records( - *close_records_stored, - *received_payment_count, - MAX_RECORDS_COUNT, - 0, - ); + let cost = calculate_cost_for_records(&QuotingMetrics { + close_records_stored: *close_records_stored, + max_records: MAX_RECORDS_COUNT, + received_payment_count: *received_payment_count, + live_time: 0, + }); *nanos_earnt += cost; *received_payment_count += 1; } @@ -1195,12 +1229,12 @@ mod tests { let mut max_store_cost = 0; for (_peer_id, (close_records_stored, nanos_earnt, times_paid)) in peers.iter() { - let cost = calculate_cost_for_records( - *close_records_stored, - *times_paid, - MAX_RECORDS_COUNT, - 0, - ); + let cost = calculate_cost_for_records(&QuotingMetrics { + close_records_stored: *close_records_stored, + max_records: MAX_RECORDS_COUNT, + received_payment_count: *times_paid, + live_time: 0, + }); // println!("{peer_id:?}:{stats:?} with storecost to be {cost}"); received_payment_count += times_paid; if *nanos_earnt == 0 { @@ -1291,7 +1325,12 @@ mod tests { for peer in peers_in_close { if let Some(stats) = peers.get(peer) { - let store_cost = calculate_cost_for_records(stats.0, stats.2, MAX_RECORDS_COUNT, 0); + let store_cost = calculate_cost_for_records(&QuotingMetrics { + close_records_stored: stats.0, + max_records: MAX_RECORDS_COUNT, + received_payment_count: stats.2, + live_time: 0, + }); if store_cost < cheapest_cost { cheapest_cost = store_cost; payee = Some(*peer); diff --git a/sn_node/src/quote.rs b/sn_node/src/quote.rs index 4374be3182..dbd5ed2a4c 100644 --- a/sn_node/src/quote.rs +++ b/sn_node/src/quote.rs @@ -118,15 +118,9 @@ pub(crate) async fn quotes_verification(network: &Network, quotes: Vec<(PeerId, .collect(); quotes_for_nodes_duty.retain(|(peer_id, quote)| { - let cost = calculate_cost_for_records( - quote.quoting_metrics.close_records_stored, - quote.quoting_metrics.received_payment_count, - quote.quoting_metrics.max_records, - quote.quoting_metrics.live_time, - ); + let cost = calculate_cost_for_records("e.quoting_metrics); let is_same_as_expected = quote.cost == NanoTokens::from(cost); - // TODO: need to confirm the quote_metrics was signed by the peer. if !is_same_as_expected { info!("Quote from {peer_id:?} using a different quoting_metrics to achieve the claimed cost. Quote {quote:?} can only result in cost {cost:?}"); network.record_node_issues(*peer_id, NodeIssue::BadQuoting); From c6553fdd57c1751127e88669cb09f5f3becd0b01 Mon Sep 17 00:00:00 2001 From: qima Date: Thu, 4 Apr 2024 19:53:07 +0800 Subject: [PATCH 010/205] test(transfer): unit tests for PaymentQuote --- sn_transfers/src/wallet/data_payments.rs | 100 ++++++++++++++++++++++- 1 file changed, 98 insertions(+), 2 deletions(-) diff --git a/sn_transfers/src/wallet/data_payments.rs b/sn_transfers/src/wallet/data_payments.rs index 08b20c084b..fcaa05fc44 100644 --- a/sn_transfers/src/wallet/data_payments.rs +++ b/sn_transfers/src/wallet/data_payments.rs @@ -15,6 +15,9 @@ use xor_name::XorName; /// The time in seconds that a quote is valid for pub const QUOTE_EXPIRATION_SECS: u64 = 3600; +/// The margin allowed for live_time +const LIVE_TIME_MARGIN: u64 = 10; + #[derive(Clone, Serialize, Deserialize, Eq, PartialEq, custom_debug::Debug)] pub struct Payment { /// The transfers we make @@ -238,8 +241,8 @@ impl PaymentQuote { let time_diff = old_elapsed.as_secs().saturating_sub(new_elapsed.as_secs()); let live_time_diff = new_quote.quoting_metrics.live_time - old_quote.quoting_metrics.live_time; - // In theory, these two shall match, give it a margin of 10 to avoid system glitch - if live_time_diff > time_diff + 10 { + // In theory, these two shall match, give it a LIVE_TIME_MARGIN to avoid system glitch + if live_time_diff > time_diff + LIVE_TIME_MARGIN { info!("claimed live_time out of sync with the timestamp"); return false; } @@ -264,3 +267,96 @@ impl PaymentQuote { true } } + +#[cfg(test)] +mod tests { + use super::*; + + use libp2p::identity::Keypair; + use std::{thread::sleep, time::Duration}; + + #[test] + fn test_is_newer_than() { + let old_quote = PaymentQuote::zero(); + sleep(Duration::from_millis(100)); + let new_quote = PaymentQuote::zero(); + assert!(new_quote.is_newer_than(&old_quote)); + assert!(!old_quote.is_newer_than(&new_quote)); + } + + #[test] + fn test_is_signed_by_claimed_peer() { + let keypair = Keypair::generate_ed25519(); + let peer_id = keypair.public().to_peer_id(); + + let false_peer = PeerId::random(); + + let mut quote = PaymentQuote::zero(); + let bytes = PaymentQuote::bytes_for_signing( + quote.content, + quote.cost, + quote.timestamp, + "e.quoting_metrics, + ); + let signature = if let Ok(sig) = keypair.sign(&bytes) { + sig + } else { + panic!("Cannot sign the quote!"); + }; + + // Check failed with both incorrect pub_key and signature + assert!(!quote.check_is_signed_by_claimed_peer(peer_id)); + assert!(!quote.check_is_signed_by_claimed_peer(false_peer)); + + // Check failed with correct pub_key but incorrect signature + quote.pub_key = keypair.public().encode_protobuf(); + assert!(!quote.check_is_signed_by_claimed_peer(peer_id)); + assert!(!quote.check_is_signed_by_claimed_peer(false_peer)); + + // Check succeed with correct pub_key and signature, + // and failed with incorrect claimed signer (peer) + quote.signature = signature; + assert!(quote.check_is_signed_by_claimed_peer(peer_id)); + assert!(!quote.check_is_signed_by_claimed_peer(false_peer)); + + // Check failed with incorrect pub_key but correct signature + quote.pub_key = Keypair::generate_ed25519().public().encode_protobuf(); + assert!(!quote.check_is_signed_by_claimed_peer(peer_id)); + assert!(!quote.check_is_signed_by_claimed_peer(false_peer)); + } + + #[test] + fn test_historical_verify() { + let mut old_quote = PaymentQuote::zero(); + sleep(Duration::from_millis(100)); + let mut new_quote = PaymentQuote::zero(); + + // historical_verify will swap quotes to compare based on timeline automatically + assert!(new_quote.historical_verify(&old_quote)); + assert!(old_quote.historical_verify(&new_quote)); + + // Out of sequence received_payment_count shall be detected + old_quote.quoting_metrics.received_payment_count = 10; + new_quote.quoting_metrics.received_payment_count = 9; + assert!(!new_quote.historical_verify(&old_quote)); + assert!(!old_quote.historical_verify(&new_quote)); + // Reset to correct one + new_quote.quoting_metrics.received_payment_count = 11; + assert!(new_quote.historical_verify(&old_quote)); + assert!(old_quote.historical_verify(&new_quote)); + + // Out of sequence live_time shall be detected + new_quote.quoting_metrics.live_time = 10; + old_quote.quoting_metrics.live_time = 11; + assert!(!new_quote.historical_verify(&old_quote)); + assert!(!old_quote.historical_verify(&new_quote)); + // Out of margin live_time shall be detected + new_quote.quoting_metrics.live_time = 11 + LIVE_TIME_MARGIN + 1; + assert!(!new_quote.historical_verify(&old_quote)); + assert!(!old_quote.historical_verify(&new_quote)); + // Reset live_time to be within the margin + new_quote.quoting_metrics.live_time = 11 + LIVE_TIME_MARGIN - 1; + assert!(new_quote.historical_verify(&old_quote)); + assert!(old_quote.historical_verify(&new_quote)); + } +} From 285b7a4943b3a7596f3e11faf1e527b358cef487 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Thu, 4 Apr 2024 17:26:55 +0530 Subject: [PATCH 011/205] feat(network): update our listen port if it set to zero --- sn_networking/src/event.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 28add13d5a..c61e2b0e13 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -353,6 +353,17 @@ impl SwarmDriver { SwarmEvent::NewListenAddr { address, .. } => { event_string = "new listen addr"; + // update our stored port if it is configured to be 0 or None + match self.listen_port { + Some(0) | None => { + if let Some(actual_port) = get_port_from_multiaddr(&address) { + info!("Our listen port is configured as 0 or is not set. Setting it to our actual port: {actual_port}"); + self.listen_port = Some(actual_port); + } + } + _ => {} + }; + let local_peer_id = *self.swarm.local_peer_id(); let address = address.with(Protocol::P2p(local_peer_id)); @@ -581,6 +592,7 @@ impl SwarmDriver { info!(%address, %our_port, "external address: new candidate has a different port, not adding it."); } } + } else { trace!("external address: listen port not set. This has to be set if you're running a node"); } } From df57a3f6f9649502efc5aa403160dac896119abc Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Thu, 4 Apr 2024 18:07:08 +0530 Subject: [PATCH 012/205] chore: publish sn_metrics crate --- release-plz.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-plz.toml b/release-plz.toml index 3b2c631674..afcbba7a71 100644 --- a/release-plz.toml +++ b/release-plz.toml @@ -55,7 +55,7 @@ publish = true name = "sn_metrics" changelog_update = true git_release_enable = false -publish = false +publish = true [[package]] name = "sn_networking" From 84bd567816c4b5691f1ceea0a9d1255092f60483 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Thu, 4 Apr 2024 19:06:44 +0530 Subject: [PATCH 013/205] fix(protocol): pre release versions should be backward compatible --- sn_protocol/src/version.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/sn_protocol/src/version.rs b/sn_protocol/src/version.rs index 1c5a6bac24..abc959cdca 100644 --- a/sn_protocol/src/version.rs +++ b/sn_protocol/src/version.rs @@ -78,14 +78,11 @@ fn write_network_version_with_slash() -> String { // Protocol support shall be downward compatible for patch only version update. // i.e. versions of `A.B.X` shall be considered as a same protocol of `A.B` +// And any pre-release versions `A.B.C-alpha.X` shall be considered as a same protocol of `A.B.C-alpha` fn get_truncate_version_str() -> &'static str { let version_str = env!("CARGO_PKG_VERSION"); - if version_str.matches('.').count() == 2 { - match version_str.rfind('.') { - Some(pos) => &version_str[..pos], - None => version_str, - } - } else { - version_str + match version_str.rfind('.') { + Some(pos) => &version_str[..pos], + None => version_str, } } From 06fc9839a9969ea8e7909b308983f3fbb2132e05 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Fri, 5 Apr 2024 01:38:57 +0530 Subject: [PATCH 014/205] chore(deps): bump dependencies --- Cargo.lock | 586 ++++++++++++++++++--------- sn_auditor/Cargo.toml | 2 +- sn_cli/Cargo.toml | 8 +- sn_client/Cargo.toml | 4 +- sn_faucet/Cargo.toml | 2 +- sn_logging/Cargo.toml | 2 +- sn_logging/src/metrics.rs | 9 +- sn_metrics/Cargo.toml | 2 +- sn_networking/Cargo.toml | 8 +- sn_networking/src/metrics.rs | 2 +- sn_node/Cargo.toml | 12 +- sn_node_manager/Cargo.toml | 8 +- sn_node_manager/src/local.rs | 2 +- sn_peers_acquisition/Cargo.toml | 2 +- sn_protocol/Cargo.toml | 2 +- sn_service_management/Cargo.toml | 2 +- sn_service_management/src/control.rs | 2 +- sn_transfers/Cargo.toml | 8 +- 18 files changed, 426 insertions(+), 237 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cbf461ad44..a7e77508a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -240,7 +240,7 @@ dependencies = [ "proc-macro2", "quote", "syn 1.0.109", - "synstructure", + "synstructure 0.12.6", ] [[package]] @@ -372,22 +372,11 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d9a9bf8b79a749ee0b911b91b671cc2b6c670bdbc7e3dfd537576ddc94bb2a2" dependencies = [ - "http", + "http 0.2.12", "log", "url", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.2.0" @@ -405,9 +394,9 @@ dependencies = [ "bitflags 1.3.2", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", "itoa", "matchit", "memchr", @@ -431,8 +420,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "mime", "rustversion", "tower-layer", @@ -486,6 +475,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" + [[package]] name = "base64ct" version = "1.6.0" @@ -904,18 +899,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "clap" -version = "3.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" -dependencies = [ - "bitflags 1.3.2", - "clap_lex 0.2.4", - "indexmap 1.9.3", - "textwrap", -] - [[package]] name = "clap" version = "4.5.4" @@ -934,8 +917,8 @@ checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" dependencies = [ "anstream", "anstyle", - "clap_lex 0.7.0", - "strsim", + "clap_lex", + "strsim 0.11.0", ] [[package]] @@ -950,15 +933,6 @@ dependencies = [ "syn 2.0.55", ] -[[package]] -name = "clap_lex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" -dependencies = [ - "os_str_bytes", -] - [[package]] name = "clap_lex" version = "0.7.0" @@ -1116,32 +1090,6 @@ dependencies = [ "tiny-keccak", ] -[[package]] -name = "criterion" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" -dependencies = [ - "anes", - "atty", - "cast", - "ciborium", - "clap 3.2.25", - "criterion-plot", - "itertools 0.10.5", - "lazy_static", - "num-traits", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_derive", - "serde_json", - "tinytemplate", - "walkdir", -] - [[package]] name = "criterion" version = "0.5.1" @@ -1151,7 +1099,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.5.4", + "clap", "criterion-plot", "is-terminal", "itertools 0.10.5", @@ -1268,22 +1216,60 @@ dependencies = [ [[package]] name = "custom_debug" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89e0ae2c2a42be29595d05c50e3ce6096c0698a97e021c3289790f0750cc8e2" +checksum = "14e715bf0e503e909c7076c052e39dd215202e8edeb32f1c194fd630c314d256" dependencies = [ "custom_debug_derive", ] [[package]] name = "custom_debug_derive" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08a9f3941234c9f62ceaa2782974827749de9b0a8a6487275a278da068e1baf7" +checksum = "f731440b39c73910e253cb465ec1fac97732b3c7af215639881ec0c2a38f4f69" dependencies = [ + "darling", + "itertools 0.12.1", "proc-macro2", - "syn 1.0.109", - "synstructure", + "quote", + "syn 2.0.55", + "synstructure 0.13.1", +] + +[[package]] +name = "darling" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 2.0.55", +] + +[[package]] +name = "darling_macro" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.55", ] [[package]] @@ -1563,7 +1549,7 @@ dependencies = [ "quote", "rustversion", "syn 1.0.109", - "synstructure", + "synstructure 0.12.6", ] [[package]] @@ -1950,9 +1936,9 @@ dependencies = [ [[package]] name = "graphviz-rust" -version = "0.7.2" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1ec243771bd8dfe7f9a2e75e28d17c66fc3901b2182c5e0eeff067623aef32" +checksum = "8c33d03804e2ce21db5821f2beb4e54f844a8f90326e6bd99a1771dc54aef427" dependencies = [ "dot-generator", "dot-structures", @@ -1988,7 +1974,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.12", "indexmap 2.2.6", "slab", "tokio", @@ -2031,7 +2017,7 @@ dependencies = [ "base64 0.21.7", "bytes", "headers-core", - "http", + "http 0.2.12", "httpdate", "mime", "sha1", @@ -2043,7 +2029,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" dependencies = [ - "http", + "http 0.2.12", ] [[package]] @@ -2067,15 +2053,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.3.9" @@ -2201,6 +2178,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.6" @@ -2208,7 +2196,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +dependencies = [ + "bytes", + "futures-core", + "http 1.1.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -2235,8 +2246,8 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -2248,6 +2259,25 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186548d73ac615b32a73aafe38fb4f56c0d340e110e5a200bcadbaf2e199263a" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -2255,25 +2285,62 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", - "hyper", + "http 0.2.12", + "hyper 0.14.28", "rustls 0.21.10", "tokio", "tokio-rustls 0.24.1", ] +[[package]] +name = "hyper-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" +dependencies = [ + "futures-util", + "http 1.1.0", + "hyper 1.2.0", + "hyper-util", + "rustls 0.22.3", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.25.0", + "tower-service", +] + [[package]] name = "hyper-timeout" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.28", "pin-project-lite", "tokio", "tokio-io-timeout", ] +[[package]] +name = "hyper-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.2.0", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -2297,6 +2364,12 @@ dependencies = [ "cc", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.4.0" @@ -2343,7 +2416,7 @@ dependencies = [ "rtnetlink", "system-configuration", "tokio", - "windows", + "windows 0.51.1", ] [[package]] @@ -2356,8 +2429,8 @@ dependencies = [ "attohttpc", "bytes", "futures", - "http", - "hyper", + "http 0.2.12", + "hyper 0.14.28", "log", "rand", "tokio", @@ -2507,7 +2580,7 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ - "hermit-abi 0.3.9", + "hermit-abi", "libc", "windows-sys 0.52.0", ] @@ -2523,9 +2596,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] @@ -2955,7 +3028,7 @@ dependencies = [ "rcgen", "ring 0.16.20", "rustls 0.21.10", - "rustls-webpki", + "rustls-webpki 0.101.7", "thiserror", "x509-parser", "yasna", @@ -2994,7 +3067,7 @@ dependencies = [ "soketto", "tracing", "url", - "webpki-roots", + "webpki-roots 0.25.4", ] [[package]] @@ -3122,9 +3195,9 @@ checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memmap2" -version = "0.5.10" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322" dependencies = [ "libc", ] @@ -3169,8 +3242,8 @@ dependencies = [ "log", "once_cell", "rustls 0.21.10", - "rustls-webpki", - "webpki-roots", + "rustls-webpki 0.101.7", + "webpki-roots 0.25.4", ] [[package]] @@ -3194,11 +3267,26 @@ dependencies = [ "downcast", "fragile", "lazy_static", - "mockall_derive", + "mockall_derive 0.11.4", "predicates 2.1.5", "predicates-tree", ] +[[package]] +name = "mockall" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43766c2b5203b10de348ffe19f7e54564b64f3d6018ff7648d1e2d6d3a0f0a48" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "lazy_static", + "mockall_derive 0.12.1", + "predicates 3.1.0", + "predicates-tree", +] + [[package]] name = "mockall_derive" version = "0.11.4" @@ -3211,6 +3299,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "mockall_derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af7cbce79ec385a1d4f54baa90a76401eb15d9cab93685f62e7e9f942aa00ae2" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 2.0.55", +] + [[package]] name = "multer" version = "2.1.0" @@ -3220,7 +3320,7 @@ dependencies = [ "bytes", "encoding_rs", "futures-util", - "http", + "http 0.2.12", "httparse", "log", "memchr", @@ -3530,7 +3630,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.9", + "hermit-abi", "libc", ] @@ -3603,7 +3703,7 @@ checksum = "7e5e5a5c4135864099f3faafbe939eb4d7f9b80ebf68a8448da961b32a7c1275" dependencies = [ "async-trait", "futures-core", - "http", + "http 0.2.12", "opentelemetry-proto", "opentelemetry-semantic-conventions", "opentelemetry_api", @@ -3683,12 +3783,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "os_str_bytes" -version = "6.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" - [[package]] name = "overload" version = "0.1.1" @@ -3945,7 +4039,7 @@ checksum = "e0c976a60b2d7e99d6f229e414670a9b85d13ac305cc6d1e9c134de58c5aaaf6" dependencies = [ "cfg-if", "concurrent-queue", - "hermit-abi 0.3.9", + "hermit-abi", "pin-project-lite", "rustix", "tracing", @@ -3989,9 +4083,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "pprof" -version = "0.11.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196ded5d4be535690899a4631cc9f18cdc41b7ebf24a79400f46f48e49a11059" +checksum = "ef5c97c51bd34c7e742402e216abdeb44d415fbe6ae41d56b114723e953711cb" dependencies = [ "backtrace", "cfg-if", @@ -4036,7 +4130,10 @@ checksum = "68b87bfd4605926cdfefc1c3b5f8fe560e3feca9d5552cf68c466d3d8236c7e8" dependencies = [ "anstyle", "difflib", + "float-cmp", + "normalize-line-endings", "predicates-core", + "regex", ] [[package]] @@ -4474,10 +4571,10 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", - "hyper", - "hyper-rustls", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", + "hyper-rustls 0.24.2", "ipnet", "js-sys", "log", @@ -4499,7 +4596,48 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots", + "webpki-roots 0.25.4", + "winreg", +] + +[[package]] +name = "reqwest" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d66674f2b6fb864665eea7a3c1ac4e3dfacd2fda83cf6f935a612e01b0e3338" +dependencies = [ + "base64 0.21.7", + "bytes", + "futures-core", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.2.0", + "hyper-rustls 0.26.0", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls 0.22.3", + "rustls-pemfile 1.0.4", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls 0.25.0", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots 0.26.1", "winreg", ] @@ -4665,10 +4803,24 @@ checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", "ring 0.17.8", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct 0.7.1", ] +[[package]] +name = "rustls" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99008d7ad0bbbea527ec27bddbc0e432c5b87d8175178cee68d2eec9c4a1813c" +dependencies = [ + "log", + "ring 0.17.8", + "rustls-pki-types", + "rustls-webpki 0.102.2", + "subtle", + "zeroize", +] + [[package]] name = "rustls-pemfile" version = "0.2.1" @@ -4687,6 +4839,12 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pki-types" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd36cc4259e3e4514335c4a138c6b43171a8d61d8f5c9348f9fc7529416f247" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -4697,6 +4855,17 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "rustls-webpki" +version = "0.102.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" +dependencies = [ + "ring 0.17.8", + "rustls-pki-types", + "untrusted 0.9.0", +] + [[package]] name = "rustversion" version = "1.0.14" @@ -5005,18 +5174,18 @@ dependencies = [ "assert_fs", "assert_matches", "async-trait", - "clap 4.5.4", + "clap", "color-eyre", "colored", "dirs-next", "indicatif", "libp2p", "libp2p-identity", - "mockall", + "mockall 0.12.1", "nix 0.27.1", - "predicates 2.1.5", + "predicates 3.1.0", "prost 0.9.0", - "reqwest", + "reqwest 0.12.2", "semver", "serde", "serde_json", @@ -5045,7 +5214,7 @@ dependencies = [ "flate2", "lazy_static", "regex", - "reqwest", + "reqwest 0.11.27", "semver", "serde_json", "tar", @@ -5059,7 +5228,7 @@ name = "sn_auditor" version = "0.1.2" dependencies = [ "blsttc", - "clap 4.5.4", + "clap", "color-eyre", "dirs-next", "graphviz-rust", @@ -5085,15 +5254,15 @@ name = "sn_cli" version = "0.90.2" dependencies = [ "aes 0.7.5", - "base64 0.21.7", + "base64 0.22.0", "bitcoin", "block-modes", "blsttc", "bytes", "chrono", - "clap 4.5.4", + "clap", "color-eyre", - "criterion 0.5.1", + "criterion", "custom_debug", "dialoguer", "dirs-next", @@ -5104,7 +5273,7 @@ dependencies = [ "libp2p", "rand", "rayon", - "reqwest", + "reqwest 0.12.2", "rmp-serde", "serde", "sn_build_info", @@ -5138,7 +5307,7 @@ dependencies = [ "futures", "getrandom", "hex", - "itertools 0.11.0", + "itertools 0.12.1", "lazy_static", "libp2p", "libp2p-identity", @@ -5175,10 +5344,10 @@ name = "sn_faucet" version = "0.4.3" dependencies = [ "assert_fs", - "base64 0.21.7", + "base64 0.22.0", "bitcoin", "blsttc", - "clap 4.5.4", + "clap", "color-eyre", "dirs-next", "fs2", @@ -5227,7 +5396,7 @@ dependencies = [ name = "sn_metrics" version = "0.1.3" dependencies = [ - "clap 4.5.4", + "clap", "color-eyre", "dirs-next", "regex", @@ -5251,8 +5420,8 @@ dependencies = [ "futures", "getrandom", "hex", - "hyper", - "itertools 0.11.0", + "hyper 0.14.28", + "itertools 0.12.1", "libp2p", "libp2p-identity", "prometheus-client", @@ -5264,7 +5433,7 @@ dependencies = [ "sn_protocol", "sn_registers", "sn_transfers", - "strum 0.26.2", + "strum", "sysinfo", "thiserror", "tiny-keccak", @@ -5287,7 +5456,7 @@ dependencies = [ "blsttc", "bytes", "chrono", - "clap 4.5.4", + "clap", "color-eyre", "crdts", "custom_debug", @@ -5296,14 +5465,14 @@ dependencies = [ "file-rotate", "futures", "hex", - "itertools 0.11.0", + "itertools 0.12.1", "lazy_static", "libp2p", "prometheus-client", "prost 0.9.0", "rand", "rayon", - "reqwest", + "reqwest 0.12.2", "rmp-serde", "self_encryption", "serde", @@ -5317,7 +5486,7 @@ dependencies = [ "sn_registers", "sn_service_management", "sn_transfers", - "strum 0.25.0", + "strum", "tempfile", "test_utils", "thiserror", @@ -5339,7 +5508,7 @@ dependencies = [ "assert_fs", "async-trait", "blsttc", - "clap 4.5.4", + "clap", "color-eyre", "hex", "libp2p", @@ -5363,11 +5532,11 @@ dependencies = [ name = "sn_peers_acquisition" version = "0.2.8" dependencies = [ - "clap 4.5.4", + "clap", "lazy_static", "libp2p", "rand", - "reqwest", + "reqwest 0.12.2", "sn_protocol", "thiserror", "tokio", @@ -5429,7 +5598,7 @@ dependencies = [ "dirs-next", "libp2p", "libp2p-identity", - "mockall", + "mockall 0.11.4", "prost 0.9.0", "semver", "serde", @@ -5451,7 +5620,7 @@ version = "0.17.1" dependencies = [ "assert_fs", "blsttc", - "criterion 0.4.0", + "criterion", "custom_debug", "dirs-next", "eyre", @@ -5557,18 +5726,15 @@ checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" [[package]] name = "strsim" -version = "0.11.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] -name = "strum" -version = "0.25.0" +name = "strsim" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" -dependencies = [ - "strum_macros 0.25.3", -] +checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" [[package]] name = "strum" @@ -5576,20 +5742,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" dependencies = [ - "strum_macros 0.26.2", -] - -[[package]] -name = "strum_macros" -version = "0.25.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.55", + "strum_macros", ] [[package]] @@ -5613,9 +5766,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "symbolic-common" -version = "10.2.1" +version = "12.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b55cdc318ede251d0957f07afe5fed912119b8c1bc5a7804151826db999e737" +checksum = "1cccfffbc6bb3bb2d3a26cd2077f4d055f6808d266f9d4d158797a4c60510dfe" dependencies = [ "debugid", "memmap2", @@ -5625,9 +5778,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "10.2.1" +version = "12.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79be897be8a483a81fff6a3a4e195b4ac838ef73ca42d348b3f722da9902e489" +checksum = "76a99812da4020a67e76c4eb41f08c87364c14170495ff780f30dd519c221a68" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -5674,11 +5827,22 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.55", +] + [[package]] name = "sysinfo" -version = "0.29.11" +version = "0.30.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd727fc423c2060f6c92d9534cef765c65a6ed3f428a03d7def74a8c4348e666" +checksum = "4b1a378e48fb3ce3a5cf04359c456c9c98ff689bcf1c1bc6e6a31f247686f275" dependencies = [ "cfg-if", "core-foundation-sys", @@ -5686,7 +5850,7 @@ dependencies = [ "ntapi", "once_cell", "rayon", - "winapi", + "windows 0.52.0", ] [[package]] @@ -5756,12 +5920,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "textwrap" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" - [[package]] name = "thiserror" version = "1.0.58" @@ -5888,7 +6046,7 @@ name = "token_supplies" version = "0.1.46" dependencies = [ "dirs-next", - "reqwest", + "reqwest 0.11.27", "serde", "serde_json", "tokio", @@ -5956,6 +6114,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls 0.22.3", + "rustls-pki-types", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.15" @@ -6020,9 +6189,9 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", - "hyper", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-timeout", "percent-encoding", "pin-project", @@ -6052,9 +6221,9 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", - "hyper", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-timeout", "percent-encoding", "pin-project", @@ -6294,7 +6463,7 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http", + "http 0.2.12", "httparse", "log", "rand", @@ -6511,9 +6680,9 @@ dependencies = [ [[package]] name = "walkdir" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", @@ -6538,8 +6707,8 @@ dependencies = [ "futures-channel", "futures-util", "headers", - "http", - "hyper", + "http 0.2.12", + "hyper 0.14.28", "log", "mime", "mime_guess", @@ -6681,6 +6850,15 @@ version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +[[package]] +name = "webpki-roots" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "which" version = "4.4.2" @@ -6740,6 +6918,16 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +dependencies = [ + "windows-core 0.52.0", + "windows-targets 0.52.4", +] + [[package]] name = "windows-core" version = "0.51.1" diff --git a/sn_auditor/Cargo.toml b/sn_auditor/Cargo.toml index 19f15fa4be..c085639f5f 100644 --- a/sn_auditor/Cargo.toml +++ b/sn_auditor/Cargo.toml @@ -21,7 +21,7 @@ bls = { package = "blsttc", version = "8.0.1" } clap = { version = "4.2.1", features = ["derive"] } color-eyre = "~0.6" dirs-next = "~2.0.0" -graphviz-rust = "0.7.1" +graphviz-rust = "0.9.0" serde = { version = "1.0.133", features = [ "derive", "rc" ]} serde_json = "1.0.108" sn_client = { path = "../sn_client", version = "0.105.2" } diff --git a/sn_cli/Cargo.toml b/sn_cli/Cargo.toml index c28daf80b7..ea48f7655d 100644 --- a/sn_cli/Cargo.toml +++ b/sn_cli/Cargo.toml @@ -32,12 +32,12 @@ open-metrics = ["sn_client/open-metrics"] [dependencies] aes="0.7.5" -base64 = { version = "0.21.7", optional = true } +base64 = { version = "0.22.0", optional = true } bitcoin = { version = "0.31.0", optional = true} block-modes="0.8.1" bls = { package = "blsttc", version = "8.0.1" } bytes = { version = "1.0.1", features = ["serde"] } -custom_debug = "~0.5.0" +custom_debug = "~0.6.1" chrono = "~0.4.19" clap = { version = "4.2.1", features = ["derive"]} color-eyre = "~0.6" @@ -49,7 +49,7 @@ indicatif = { version = "0.17.5", features = ["tokio"] } libp2p = { version="0.53", features = ["identify", "kad"] } rand = "0.8.5" rayon = "1.8.0" -reqwest = { version="0.11.18", default-features=false, features = ["rustls"] } +reqwest = { version="0.12.2", default-features=false, features = ["rustls-tls-manual-roots"] } rmp-serde = "1.1.1" serde = { version = "1.0.133", features = [ "derive"]} sn_build_info = { path="../sn_build_info", version = "0.1.5" } @@ -62,7 +62,7 @@ tiny-keccak = "~2.0.2" tokio = { version = "1.32.0", features = ["io-util", "macros", "parking_lot", "rt", "sync", "time", "fs"] } tracing = { version = "~0.1.26" } url = "2.4.0" -walkdir = "~2.4.0" +walkdir = "~2.5.0" xor_name = "5.0.0" [dev-dependencies] diff --git a/sn_client/Cargo.toml b/sn_client/Cargo.toml index d7504fb340..9fbf5532a8 100644 --- a/sn_client/Cargo.toml +++ b/sn_client/Cargo.toml @@ -24,10 +24,10 @@ backoff = { version = "0.4.0", features = ["tokio"] } bls = { package = "blsttc", version = "8.0.1" } bytes = { version = "1.0.1", features = ["serde"] } crdts = "7.3.2" -custom_debug = "~0.5.0" +custom_debug = "~0.6.1" futures = "~0.3.13" hex = "~0.4.3" -itertools = "~0.11.0" +itertools = "~0.12.1" libp2p = { version="0.53", features = ["identify"] } petgraph = { version = "0.6.4", features = ["serde-1"] } prometheus-client = { version = "0.22", optional = true } diff --git a/sn_faucet/Cargo.toml b/sn_faucet/Cargo.toml index 545ff5b8b5..cdab9a8378 100644 --- a/sn_faucet/Cargo.toml +++ b/sn_faucet/Cargo.toml @@ -21,7 +21,7 @@ name = "faucet" [dependencies] warp = "0.3" assert_fs = "1.0.0" -base64 = { version = "0.21.7", optional = true } +base64 = { version = "0.22.0", optional = true } bitcoin = { version = "0.31.0", features = [ "rand-std", "base64", diff --git a/sn_logging/Cargo.toml b/sn_logging/Cargo.toml index 9660c48144..f79f0b92b5 100644 --- a/sn_logging/Cargo.toml +++ b/sn_logging/Cargo.toml @@ -20,7 +20,7 @@ opentelemetry-semantic-conventions = { version = "0.12.0", optional = true } rand = { version = "~0.8.5", features = ["small_rng"], optional = true } serde = { version = "1.0.133", features = [ "derive", "rc" ], optional = true } serde_json = {version = "1.0", optional = true } -sysinfo = { version = "0.29.0", default-features = false, optional = true } +sysinfo = { version = "0.30.8", default-features = false, optional = true } thiserror = "1.0.23" tokio = { version = "1.32.0", optional = true } tracing = { version = "~0.1.26" } diff --git a/sn_logging/src/metrics.rs b/sn_logging/src/metrics.rs index 6cc7b4848d..cce2659846 100644 --- a/sn_logging/src/metrics.rs +++ b/sn_logging/src/metrics.rs @@ -8,7 +8,7 @@ use serde::Serialize; use std::time::Duration; -use sysinfo::{self, CpuExt, Pid, PidExt, ProcessExt, System, SystemExt}; +use sysinfo::{self, Networks, Pid, System}; use tracing::{debug, error}; const UPDATE_INTERVAL: Duration = Duration::from_secs(15); @@ -47,10 +47,11 @@ struct ProcessMetrics { // The function should be spawned as a task and should be re-run if our main process is restarted. pub async fn init_metrics(pid: u32) { let mut sys = System::new_all(); + let mut networks = Networks::new_with_refreshed_list(); let pid = Pid::from_u32(pid); loop { - refresh_metrics(&mut sys, pid); + refresh_metrics(&mut sys, &mut networks, pid); let process = match sys.process(pid) { Some(safenode) => { @@ -87,9 +88,9 @@ pub async fn init_metrics(pid: u32) { } // Refreshes only the metrics that we interested in. -fn refresh_metrics(sys: &mut System, pid: Pid) { +fn refresh_metrics(sys: &mut System, networks: &mut Networks, pid: Pid) { sys.refresh_process(pid); sys.refresh_memory(); - sys.refresh_networks(); sys.refresh_cpu(); + networks.refresh(); } diff --git a/sn_metrics/Cargo.toml b/sn_metrics/Cargo.toml index b0509a6ced..2fe4db2a56 100644 --- a/sn_metrics/Cargo.toml +++ b/sn_metrics/Cargo.toml @@ -22,4 +22,4 @@ regex = "1.10" serde = { version = "1.0.133", features = ["derive"] } serde_yaml = "0.9.25" url = "2.4.1" -walkdir = "~2.4.0" +walkdir = "~2.5" diff --git a/sn_networking/Cargo.toml b/sn_networking/Cargo.toml index 270f0f7999..ef985f527c 100644 --- a/sn_networking/Cargo.toml +++ b/sn_networking/Cargo.toml @@ -29,8 +29,8 @@ hyper = { version = "0.14", features = [ "tcp", "http1", ], optional = true } -itertools = "~0.11.0" -custom_debug = "~0.5.0" +itertools = "~0.12.1" +custom_debug = "~0.6.1" libp2p = { version = "0.53", features = [ "tokio", "dns", @@ -52,7 +52,7 @@ serde = { version = "1.0.133", features = ["derive", "rc"] } sn_protocol = { path = "../sn_protocol", version = "0.16.1" } sn_transfers = { path = "../sn_transfers", version = "0.17.1" } sn_registers = { path = "../sn_registers", version = "0.3.12" } -sysinfo = { version = "0.29.0", default-features = false, optional = true } +sysinfo = { version = "0.30.8", default-features = false, optional = true } thiserror = "1.0.23" tiny-keccak = { version = "~2.0.2", features = ["sha3"] } tokio = { version = "1.32.0", features = [ @@ -66,7 +66,7 @@ tracing = { version = "~0.1.26" } xor_name = "5.0.0" backoff = { version = "0.4.0", features = ["tokio"] } aes-gcm-siv = "0.11.1" -walkdir = "~2.4.0" +walkdir = "~2.5.0" strum = { version = "0.26.2", features = ["derive"] } [dev-dependencies] diff --git a/sn_networking/src/metrics.rs b/sn_networking/src/metrics.rs index 0c20050a89..ad24e1becf 100644 --- a/sn_networking/src/metrics.rs +++ b/sn_networking/src/metrics.rs @@ -9,7 +9,7 @@ use crate::target_arch::sleep; use libp2p::metrics::{Metrics as Libp2pMetrics, Recorder}; use prometheus_client::{metrics::gauge::Gauge, registry::Registry}; -use sysinfo::{Pid, PidExt, ProcessExt, ProcessRefreshKind, System, SystemExt}; +use sysinfo::{Pid, ProcessRefreshKind, System}; use tokio::time::Duration; const UPDATE_INTERVAL: Duration = Duration::from_secs(15); diff --git a/sn_node/Cargo.toml b/sn_node/Cargo.toml index 0e6e61856d..89356652f3 100644 --- a/sn_node/Cargo.toml +++ b/sn_node/Cargo.toml @@ -30,13 +30,13 @@ bytes = { version = "1.0.1", features = ["serde"] } clap = { version = "4.2.1", features = ["derive"] } crdts = { version = "7.3", default-features = false, features = ["merkle"] } chrono = "~0.4.19" -custom_debug = "~0.5.0" +custom_debug = "~0.6.1" dirs-next = "~2.0.0" eyre = "0.6.8" file-rotate = "0.7.3" futures = "~0.3.13" hex = "~0.4.3" -itertools = "~0.11.0" +itertools = "~0.12.1" lazy_static = "~1.4.0" libp2p = { version = "0.53", features = ["tokio", "dns", "kad", "macros"] } prometheus-client = { version = "0.22", optional = true } @@ -73,15 +73,15 @@ tracing = { version = "~0.1.26" } tracing-appender = "~0.2.0" tracing-opentelemetry = { version = "0.21", optional = true } tracing-subscriber = { version = "0.3.16" } -walkdir = "~2.4.0" +walkdir = "~2.5.0" xor_name = "5.0.0" -strum = { version = "0.25.0", features = ["derive"] } +strum = { version = "0.26.2", features = ["derive"] } color-eyre = "0.6.2" [dev-dependencies] assert_matches = "1.5.0" -reqwest = { version = "0.11.18", default-features = false, features = [ - "rustls", +reqwest = { version = "0.12.2", default-features = false, features = [ + "rustls-tls-manual-roots", ] } serde_json = "1.0" sn_protocol = { path = "../sn_protocol", version = "0.16.1", features = [ diff --git a/sn_node_manager/Cargo.toml b/sn_node_manager/Cargo.toml index 11741b7aaa..8c51e501fa 100644 --- a/sn_node_manager/Cargo.toml +++ b/sn_node_manager/Cargo.toml @@ -46,7 +46,7 @@ sn_protocol = { path = "../sn_protocol", version = "0.16.1" } sn_service_management = { path = "../sn_service_management", version = "0.2.1" } sn-releases = "0.2.0" sn_transfers = { path = "../sn_transfers", version = "0.17.1" } -sysinfo = "0.29.10" +sysinfo = "0.30.8" tokio = { version = "1.26", features = ["full"] } tracing = { version = "~0.1.26" } # watch out updating this, protoc compiler needs to be installed on all build systems @@ -64,9 +64,9 @@ assert_cmd = "2.0.12" assert_fs = "1.0.13" assert_matches = "1.5.0" async-trait = "0.1" -mockall = "0.11.3" -reqwest = { version = "0.11", default-features = false, features = [ +mockall = "0.12.1" +reqwest = { version = "0.12", default-features = false, features = [ "json", "rustls-tls", ] } -predicates = "2.0" +predicates = "3.1.0" diff --git a/sn_node_manager/src/local.rs b/sn_node_manager/src/local.rs index c9151d5ece..6f2a9d3dff 100644 --- a/sn_node_manager/src/local.rs +++ b/sn_node_manager/src/local.rs @@ -25,7 +25,7 @@ use std::{ process::{Command, Stdio}, str::FromStr, }; -use sysinfo::{Pid, ProcessExt, System, SystemExt}; +use sysinfo::{Pid, System}; #[cfg_attr(test, automock)] pub trait Launcher { diff --git a/sn_peers_acquisition/Cargo.toml b/sn_peers_acquisition/Cargo.toml index d0ed166892..a59695b7c0 100644 --- a/sn_peers_acquisition/Cargo.toml +++ b/sn_peers_acquisition/Cargo.toml @@ -20,7 +20,7 @@ clap = { version = "4.2.1", features = ["derive", "env"] } lazy_static = "~1.4.0" libp2p = { version="0.53", features = [] } rand = "0.8.5" -reqwest = { version="0.11.18", default-features=false, features = ["rustls-tls"], optional = true } +reqwest = { version="0.12.2", default-features=false, features = ["rustls-tls"], optional = true } sn_protocol = { path = "../sn_protocol", version = "0.16.1" } thiserror = "1.0.23" tokio = { version = "1.32.0", optional = true, default-features = false} diff --git a/sn_protocol/Cargo.toml b/sn_protocol/Cargo.toml index 81f7e3af11..f2aa7a1cf4 100644 --- a/sn_protocol/Cargo.toml +++ b/sn_protocol/Cargo.toml @@ -19,7 +19,7 @@ bls = { package = "blsttc", version = "8.0.1" } bytes = { version = "1.0.1", features = ["serde"] } color-eyre = "0.6.2" crdts = { version = "7.3", default-features = false, features = ["merkle"] } -custom_debug = "~0.5.0" +custom_debug = "~0.6.1" dirs-next = "~2.0.0" hex = "~0.4.3" lazy_static = "~1.4.0" diff --git a/sn_service_management/Cargo.toml b/sn_service_management/Cargo.toml index 41c1c1cd31..aea1beea99 100644 --- a/sn_service_management/Cargo.toml +++ b/sn_service_management/Cargo.toml @@ -20,7 +20,7 @@ serde_json = "1.0" semver = "1.0.20" service-manager = "0.6.0" sn_protocol = { path = "../sn_protocol", version = "0.16.1", features = ["rpc"] } -sysinfo = "0.29.10" +sysinfo = "0.30.8" thiserror = "1.0.23" tokio = { version = "1.32.0", features = ["time"] } tonic = { version = "0.6.2" } diff --git a/sn_service_management/src/control.rs b/sn_service_management/src/control.rs index 41afe67232..11bc2e7218 100644 --- a/sn_service_management/src/control.rs +++ b/sn_service_management/src/control.rs @@ -13,7 +13,7 @@ use service_manager::{ ServiceUninstallCtx, }; use std::net::{SocketAddr, TcpListener}; -use sysinfo::{Pid, ProcessExt, System, SystemExt}; +use sysinfo::{Pid, System}; /// A thin wrapper around the `service_manager::ServiceManager`, which makes our own testing /// easier. diff --git a/sn_transfers/Cargo.toml b/sn_transfers/Cargo.toml index d021324796..483b24acd2 100644 --- a/sn_transfers/Cargo.toml +++ b/sn_transfers/Cargo.toml @@ -12,7 +12,7 @@ version = "0.17.1" [dependencies] bls = { package = "blsttc", version = "8.0.1" } -custom_debug = "~0.5.0" +custom_debug = "~0.6.1" dirs-next = "~2.0.0" hex = "~0.4.3" lazy_static = "~1.4.0" @@ -24,13 +24,13 @@ serde_json = "1.0.108" thiserror = "1.0.24" tiny-keccak = { version = "~2.0.2", features = [ "sha3" ] } tracing = { version = "~0.1.26" } -walkdir = "~2.4.0" +walkdir = "~2.5.0" xor_name = "5.0.0" rayon = "1.8.0" [dev-dependencies] tokio = { version = "1.32.0", features = ["macros", "rt"] } -criterion = "0.4.0" +criterion = "0.5.1" assert_fs = "1.0.0" eyre = "0.6.8" @@ -39,7 +39,7 @@ eyre = "0.6.8" fs2 = "0.4.3" [target."cfg(unix)".dev-dependencies.pprof] -version = "0.11.0" +version = "0.13.0" features = [ "flamegraph" ] [[bench]] From 131efe1c7a08850c2391a042dd27c15cf5139e1d Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Fri, 5 Apr 2024 01:43:32 +0530 Subject: [PATCH 015/205] fix(network): clients should not perform farthest relevant record check --- sn_networking/src/driver.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index c9f0148b95..42d5a2c5cd 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -602,14 +602,15 @@ impl SwarmDriver { } } _ = set_farthest_record_interval.tick() => { - let closest_k_peers = self - .get_closest_k_value_local_peers(); - - if let Some(distance) = self.get_farthest_relevant_address_estimate(&closest_k_peers) { - // set any new distance to fathest record in the store - self.swarm.behaviour_mut().kademlia.store_mut().set_distance_range(distance); - // the distance range within the replication_fetcher shall be in sync as well - self.replication_fetcher.set_distance_range(distance); + if !self.is_client { + let closest_k_peers = self.get_closest_k_value_local_peers(); + + if let Some(distance) = self.get_farthest_relevant_address_estimate(&closest_k_peers) { + // set any new distance to farthest record in the store + self.swarm.behaviour_mut().kademlia.store_mut().set_distance_range(distance); + // the distance range within the replication_fetcher shall be in sync as well + self.replication_fetcher.set_distance_range(distance); + } } } } From 52d9673f6217674ac26e54d1274de5daec9fd058 Mon Sep 17 00:00:00 2001 From: bochaco Date: Tue, 2 Apr 2024 15:10:16 -0300 Subject: [PATCH 016/205] feat(register): expose API to get children of an entry --- sn_registers/src/reg_crdt.rs | 63 ++++++++++++++++++++++++++++++++++++ sn_registers/src/register.rs | 5 +++ 2 files changed, 68 insertions(+) diff --git a/sn_registers/src/reg_crdt.rs b/sn_registers/src/reg_crdt.rs index 254c156b59..844b3bfce3 100644 --- a/sn_registers/src/reg_crdt.rs +++ b/sn_registers/src/reg_crdt.rs @@ -113,6 +113,15 @@ impl RegisterCrdt { .collect() } + /// Returns the children of an entry, along with their corresponding entry hashes + pub fn children(&self, hash: &EntryHash) -> BTreeSet<(EntryHash, Entry)> { + self.data + .children(hash.0) + .hashes_and_nodes() + .map(|(hash, node)| (EntryHash(hash), node.value.clone())) + .collect() + } + /// Access the underlying MerkleReg (e.g. for access to history) /// NOTE: This API is unstable and may be removed in the future pub(crate) fn merkle_reg(&self) -> &MerkleReg { @@ -163,4 +172,58 @@ mod tests { Ok(()) } + + #[test] + fn entry_children() -> Result<()> { + let mut rng = rand::thread_rng(); + let address = RegisterAddress { + meta: XorName::random(&mut rng), + owner: SecretKey::random().public_key(), + }; + let mut crdt = RegisterCrdt::new(address); + + // let's build the following entries hierarchy to test: + // - entry_1 has no child + // - entry_2_1, entry_2_2, and entry_2_3, all have entry_1 as child + // - entry_3 has both entry_2_1 and entry_2_2 as children + let entry_1 = vec![0x0, 0x1]; + let entry_2_1 = vec![0x2, 0x1]; + let entry_2_2 = vec![0x2, 0x2]; + let entry_2_3 = vec![0x2, 0x3]; + let entry_3 = vec![0x0, 0x3]; + let (entry_hash_1, _, _) = crdt.write(entry_1.clone(), &BTreeSet::new())?; + let (entry_hash_2_1, _, _) = + crdt.write(entry_2_1.clone(), &[entry_hash_1].into_iter().collect())?; + let (entry_hash_2_2, _, _) = + crdt.write(entry_2_2.clone(), &[entry_hash_1].into_iter().collect())?; + let (entry_hash_2_3, _, _) = + crdt.write(entry_2_3.clone(), &[entry_hash_1].into_iter().collect())?; + let (entry_hash_3, _, _) = crdt.write( + entry_3, + &[entry_hash_2_1, entry_hash_2_2].into_iter().collect(), + )?; + + let children_entry_1 = crdt.children(&entry_hash_1); + assert_eq!(children_entry_1, BTreeSet::new()); + + let children_entry_2_1 = crdt.children(&entry_hash_2_1); + let children_entry_2_2 = crdt.children(&entry_hash_2_2); + let children_entry_2_3 = crdt.children(&entry_hash_2_3); + assert_eq!( + children_entry_2_1, + [(entry_hash_1, entry_1)].into_iter().collect() + ); + assert_eq!(children_entry_2_1, children_entry_2_2); + assert_eq!(children_entry_2_1, children_entry_2_3); + + let children_entry_3 = crdt.children(&entry_hash_3); + assert_eq!( + children_entry_3, + [(entry_hash_2_1, entry_2_1), (entry_hash_2_2, entry_2_2)] + .into_iter() + .collect() + ); + + Ok(()) + } } diff --git a/sn_registers/src/register.rs b/sn_registers/src/register.rs index 02bb61758d..7e590a3972 100644 --- a/sn_registers/src/register.rs +++ b/sn_registers/src/register.rs @@ -193,6 +193,11 @@ impl Register { self.crdt.read() } + /// Returns the children of an entry, along with their corresponding entry hashes + pub fn children(&self, hash: &EntryHash) -> BTreeSet<(EntryHash, Entry)> { + self.crdt.children(hash) + } + /// Return the permission. pub fn permissions(&self) -> &Permissions { &self.permissions From 65924b83b2ea7e1dc66784de7006dbb5a5984adc Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 4 Apr 2024 19:33:57 +0900 Subject: [PATCH 017/205] fix: orphan parent bug, improve fault detection and logging --- sn_auditor/src/dag_db.rs | 22 ++++- sn_client/src/audit.rs | 25 ++--- sn_client/src/audit/dag_error.rs | 6 ++ sn_client/src/audit/spend_dag.rs | 112 ++++++++++++++-------- sn_client/src/audit/spend_dag_building.rs | 24 +++-- 5 files changed, 125 insertions(+), 64 deletions(-) diff --git a/sn_auditor/src/dag_db.rs b/sn_auditor/src/dag_db.rs index 98a11e4a0c..94759c7497 100644 --- a/sn_auditor/src/dag_db.rs +++ b/sn_auditor/src/dag_db.rs @@ -82,7 +82,7 @@ impl SpendDagDb { let (spend_type, spends) = match spend { SpendDagGet::SpendNotFound => ("SpendNotFound", vec![]), - SpendDagGet::SpendIsAnUtxo => ("SpendIsAnUtxo", vec![]), + SpendDagGet::SpendKeyExists => ("SpendKeyExists", vec![]), SpendDagGet::DoubleSpend(vs) => ("DoubleSpend", vs), SpendDagGet::Spend(s) => ("Spend", vec![*s]), }; @@ -176,7 +176,7 @@ impl SpendDagDb { let mut w_handle = dag_ref .write() .map_err(|e| eyre!("Failed to get write lock: {e}"))?; - w_handle.merge(other); + w_handle.merge(other)?; Ok(()) } } @@ -218,17 +218,31 @@ fn dag_to_svg(dag: &SpendDag) -> Result> { // - makes spends clickable // - spend address reveals on hover // - marks poisoned spends as red +// - marks UTXOs and unknown ancestors as yellow // - just pray it works on windows fn quick_edit_svg(svg: Vec, dag: &SpendDag) -> Result> { let mut str = String::from_utf8(svg).map_err(|err| eyre!("Failed svg conversion: {err}"))?; let spend_addrs: Vec<_> = dag.all_spends().iter().map(|s| s.address()).collect(); let utxo_addrs = dag.get_utxos(); + let unknown_parents = dag.get_unknown_parents(); + let all_addrs = spend_addrs + .iter() + .chain(utxo_addrs.iter()) + .chain(unknown_parents.iter()); - for addr in spend_addrs.iter().chain(utxo_addrs.iter()) { + for addr in all_addrs { let addr_hex = addr.to_hex().to_string(); let is_fault = !dag.get_spend_faults(addr).is_empty(); - let colour = if is_fault { "red" } else { "none" }; + let is_known_but_not_gathered = matches!(dag.get_spend(addr), SpendDagGet::SpendKeyExists); + let colour = if is_fault { + "red" + } else if is_known_but_not_gathered { + "yellow" + } else { + "none" + }; + let link = format!(""); let idxs = dag.get_spend_indexes(addr); for i in idxs { diff --git a/sn_client/src/audit.rs b/sn_client/src/audit.rs index 8f81eb2669..6eac78efa5 100644 --- a/sn_client/src/audit.rs +++ b/sn_client/src/audit.rs @@ -20,7 +20,7 @@ use super::{ use futures::future::join_all; use sn_networking::{target_arch::Instant, GetRecordError, NetworkError}; -use sn_transfers::{SignedSpend, SpendAddress, WalletError, WalletResult}; +use sn_transfers::{Hash, SignedSpend, SpendAddress, WalletError, WalletResult}; use std::{collections::BTreeSet, iter::Iterator}; impl Client { @@ -73,17 +73,19 @@ impl Client { let parent_tx_hash = parent_tx.hash(); let parent_keys = parent_tx.inputs.iter().map(|input| input.unique_pubkey); let addrs_to_verify = parent_keys.map(|k| SpendAddress::from_unique_pubkey(&k)); - debug!("Depth {depth} - Verifying parent Tx : {parent_tx_hash:?}"); + debug!("Depth {depth} - Verifying parent Tx : {parent_tx_hash:?} with inputs: {addrs_to_verify:?}"); // get all parent spends in parallel let tasks: Vec<_> = addrs_to_verify - .into_iter() + .clone() .map(|a| self.get_spend_from_network(a)) .collect(); let spends = join_all(tasks).await .into_iter() - .collect::>>() - .map_err(|err| WalletError::CouldNotVerifyTransfer(format!("at depth {depth} - Failed to get spends from network for parent Tx {parent_tx_hash:?}: {err}")))?; + .zip(addrs_to_verify.into_iter()) + .map(|(maybe_spend, a)| + maybe_spend.map_err(|err| WalletError::CouldNotVerifyTransfer(format!("at depth {depth} - Failed to get spend {a:?} from network for parent Tx {parent_tx_hash:?}: {err}")))) + .collect::>>()?; debug!( "Depth {depth} - Got {:?} spends for parent Tx: {parent_tx_hash:?}", spends.len() @@ -184,7 +186,7 @@ impl Client { .iter() .map(|output| output.unique_pubkey); let addrs_to_follow = descendant_keys.map(|k| SpendAddress::from_unique_pubkey(&k)); - debug!("Gen {gen} - Following descendant Tx : {descendant_tx_hash:?}"); + debug!("Gen {gen} - Following descendant Tx : {descendant_tx_hash:?} with outputs: {addrs_to_follow:?}"); // get all descendant spends in parallel let tasks: Vec<_> = addrs_to_follow @@ -198,8 +200,7 @@ impl Client { .collect::>(); // split spends into utxos and spends - let (utxos, spends) = split_utxos_and_spends(spends_res) - .map_err(|err| WalletError::CouldNotVerifyTransfer(format!("at gen {gen} - Failed to get spends from network for descendant Tx {descendant_tx_hash:?}: {err}")))?; + let (utxos, spends) = split_utxos_and_spends(spends_res, gen, descendant_tx_hash)?; debug!("Gen {gen} - Got {:?} spends and {:?} utxos for descendant Tx: {descendant_tx_hash:?}", spends.len(), utxos.len()); trace!("Spends for {descendant_tx_hash:?} - {spends:?}"); next_gen_utxos.extend(utxos); @@ -241,7 +242,9 @@ impl Client { fn split_utxos_and_spends( spends_res: Vec<(Result, SpendAddress)>, -) -> Result<(Vec, Vec)> { + gen: usize, + descendant_tx_hash: Hash, +) -> WalletResult<(Vec, Vec)> { let mut utxos = Vec::new(); let mut spends = Vec::new(); @@ -254,8 +257,8 @@ fn split_utxos_and_spends( utxos.push(addr); } Err(err) => { - warn!("Error while following spends: {err}"); - return Err(err); + warn!("Error while following spend {addr:?}: {err}"); + return Err(WalletError::CouldNotVerifyTransfer(format!("at gen {gen} - Failed to get spend {addr:?} from network for descendant Tx {descendant_tx_hash:?}: {err}"))); } } } diff --git a/sn_client/src/audit/dag_error.rs b/sn_client/src/audit/dag_error.rs index 7423ffdb47..d553ed22ee 100644 --- a/sn_client/src/audit/dag_error.rs +++ b/sn_client/src/audit/dag_error.rs @@ -40,6 +40,11 @@ pub enum SpendFault { }, #[error("Invalid transaction for spend at {0:?}: {1}")] InvalidTransaction(SpendAddress, String), + #[error("Spend at {addr:?} has an unknown ancestor at {ancestor_addr:?}, until this ancestor is added to the DAG, it cannot be verified")] + UnknownAncestor { + addr: SpendAddress, + ancestor_addr: SpendAddress, + }, #[error("Poisoned ancestry for spend at {0:?}: {1}")] PoisonedAncestry(SpendAddress, String), #[error("Spend at {addr:?} does not descend from given source: {src:?}")] @@ -66,6 +71,7 @@ impl SpendFault { | SpendFault::MissingAncestry { addr, .. } | SpendFault::InvalidAncestry { addr, .. } | SpendFault::InvalidTransaction(addr, _) + | SpendFault::UnknownAncestor { addr, .. } | SpendFault::PoisonedAncestry(addr, _) | SpendFault::OrphanSpend { addr, .. } => *addr, } diff --git a/sn_client/src/audit/spend_dag.rs b/sn_client/src/audit/spend_dag.rs index 197f1d73a8..2d63d90b3f 100644 --- a/sn_client/src/audit/spend_dag.rs +++ b/sn_client/src/audit/spend_dag.rs @@ -46,7 +46,7 @@ type DagIndex = usize; /// Internal Dag entry type #[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)] enum DagEntry { - Utxo(DagIndex), + NotGatheredYet(DagIndex), DoubleSpend(Vec<(SignedSpend, DagIndex)>), Spend(Box, DagIndex), } @@ -54,7 +54,7 @@ enum DagEntry { impl DagEntry { fn indexes(&self) -> Vec { match self { - DagEntry::Utxo(idx) => vec![*idx], + DagEntry::NotGatheredYet(idx) => vec![*idx], DagEntry::DoubleSpend(spends) => spends.iter().map(|(_, idx)| *idx).collect(), DagEntry::Spend(_, idx) => vec![*idx], } @@ -64,7 +64,7 @@ impl DagEntry { match self { DagEntry::Spend(spend, _) => vec![&**spend], DagEntry::DoubleSpend(spends) => spends.iter().map(|(s, _)| s).collect(), - DagEntry::Utxo(_) => vec![], + DagEntry::NotGatheredYet(_) => vec![], } } } @@ -74,8 +74,8 @@ impl DagEntry { pub enum SpendDagGet { /// Spend does not exist in the DAG SpendNotFound, - /// Spend is an UTXO, meaning it was not spent yet but its ancestors exist - SpendIsAnUtxo, + /// Spend key is refered to by known spends but does not exist in the DAG yet + SpendKeyExists, /// Spend is a double spend DoubleSpend(Vec), /// Spend is in the DAG @@ -127,7 +127,7 @@ impl SpendDag { node_idx } // or upgrade existing utxo to spend - Some(DagEntry::Utxo(idx)) => { + Some(DagEntry::NotGatheredYet(idx)) => { self.spends .insert(spend_addr, DagEntry::Spend(Box::new(spend.clone()), idx)); NodeIndex::new(idx) @@ -168,7 +168,7 @@ impl SpendDag { // add descendant if not already in dag let spends_at_addr = self.spends.entry(descendant_addr).or_insert_with(|| { let node_idx = self.dag.add_node(descendant_addr); - DagEntry::Utxo(node_idx.index()) + DagEntry::NotGatheredYet(node_idx.index()) }); // link to descendant @@ -192,7 +192,7 @@ impl SpendDag { // add ancestor if not already in dag let spends_at_addr = self.spends.entry(ancestor_addr).or_insert_with(|| { let node_idx = self.dag.add_node(ancestor_addr); - DagEntry::Utxo(node_idx.index()) + DagEntry::NotGatheredYet(node_idx.index()) }); // link to ancestor @@ -206,6 +206,27 @@ impl SpendDag { true } + /// Get the unknown parents: all the addresses that are refered to as parents by other spends + /// but don't have parents themselves. + /// Those Spends must exist somewhere on the Network, we just haven't gathered them yet. + pub fn get_unknown_parents(&self) -> BTreeSet { + let mut sources = BTreeSet::new(); + for node_index in self.dag.node_indices() { + if !self + .dag + .neighbors_directed(node_index, petgraph::Direction::Incoming) + .any(|_| true) + { + let utxo_addr = self.dag[node_index]; + sources.insert(utxo_addr); + } + } + sources + } + + /// Get the UTXOs: all the addresses that are refered to as children by other spends + /// but that don't have children themselves. + /// Those will eventually exist on the Network as the address is spent by their owners. pub fn get_utxos(&self) -> BTreeSet { let mut leaves = BTreeSet::new(); for node_index in self.dag.node_indices() { @@ -226,18 +247,19 @@ impl SpendDag { } /// Merges the given dag into ours - pub fn merge(&mut self, sub_dag: SpendDag) { + pub fn merge(&mut self, sub_dag: SpendDag) -> Result<(), DagError> { + let source = self.source(); info!( "Merging sub DAG starting at {:?} into our DAG with source {:?}", sub_dag.source(), - self.source() + source ); for (addr, spends) in sub_dag.spends { - // only add spends to the dag, ignoring utxos + // only add spends to the dag, ignoring utxos and not yet gathered relatives // utxos will be added automatically as their ancestors are added // edges are updated by the insert method match spends { - DagEntry::Utxo(_) => continue, + DagEntry::NotGatheredYet(_) => continue, DagEntry::DoubleSpend(spends) => { for (spend, _) in spends { self.insert(addr, spend); @@ -249,15 +271,15 @@ impl SpendDag { } } - // merge errors - self.faults.extend(sub_dag.faults); + // recompute faults + self.record_faults(&source) } /// Get the spend at a given address pub fn get_spend(&self, addr: &SpendAddress) -> SpendDagGet { match self.spends.get(addr) { None => SpendDagGet::SpendNotFound, - Some(DagEntry::Utxo(_)) => SpendDagGet::SpendIsAnUtxo, + Some(DagEntry::NotGatheredYet(_)) => SpendDagGet::SpendKeyExists, Some(DagEntry::DoubleSpend(spends)) => { SpendDagGet::DoubleSpend(spends.iter().map(|(s, _)| s.clone()).collect()) } @@ -314,11 +336,11 @@ impl SpendDag { Some(DagEntry::Spend(ancestor_spend, _)) => { ancestors.insert(*ancestor_spend.clone()); } - Some(DagEntry::Utxo(_)) => { - warn!("InvalidAncestry: SpendIsAnUtxo ancestor {ancestor_addr:?} for spend {spend:?}"); - return Err(SpendFault::InvalidAncestry { + Some(DagEntry::NotGatheredYet(_)) => { + warn!("UnknownAncestor: ancestor {ancestor_addr:?} was not gathered yet for spend {spend:?}"); + return Err(SpendFault::UnknownAncestor { addr, - invalid_ancestor: ancestor_addr, + ancestor_addr, }); } Some(DagEntry::DoubleSpend(_)) => { @@ -329,10 +351,11 @@ impl SpendDag { }); } None => { + warn!("MissingAncestry: ancestor {ancestor_addr:?} is unknown for spend {spend:?}"); return Err(SpendFault::MissingAncestry { addr, invalid_ancestor: ancestor_addr, - }) + }); } } } @@ -345,8 +368,8 @@ impl SpendDag { let mut to_traverse = BTreeSet::from_iter(vec![addr]); while let Some(current_addr) = to_traverse.pop_first() { // get the spend at this address - let indexes = match self.spends.get(current_addr) { - Some(entry) => entry.indexes(), + let dag_entry = match self.spends.get(current_addr) { + Some(entry) => entry, None => { warn!("Incoherent DAG, missing descendant spend when expecting one at: {current_addr:?}"); return Err(DagError::IncoherentDag( @@ -355,6 +378,14 @@ impl SpendDag { )); } }; + let (spends, indexes) = (dag_entry.spends(), dag_entry.indexes()); + + // get descendants via Tx data + let descendants_via_tx: BTreeSet = spends + .into_iter() + .flat_map(|s| s.spend.spent_tx.outputs.to_vec()) + .map(|o| SpendAddress::from_unique_pubkey(&o.unique_pubkey)) + .collect(); // get descendants via DAG let descendants_via_dag: BTreeSet<&SpendAddress> = indexes @@ -366,24 +397,17 @@ impl SpendDag { }) .collect(); - // get descendants via Tx data - let descendants_via_tx: BTreeSet = self - .spends - .get(current_addr) - .map(|entry| entry.spends()) - .unwrap_or_default() - .into_iter() - .flat_map(|s| s.spend.spent_tx.outputs.to_vec()) - .map(|o| SpendAddress::from_unique_pubkey(&o.unique_pubkey)) - .collect(); - // report inconsistencies if descendants_via_dag != descendants_via_tx.iter().collect() { - warn!("Incoherent DAG at: {current_addr:?}"); - return Err(DagError::IncoherentDag( - *current_addr, - format!("descendants via DAG: {descendants_via_dag:?} do not match descendants via TX: {descendants_via_tx:?}") - )); + if matches!(dag_entry, DagEntry::NotGatheredYet(_)) { + debug!("Spend at {current_addr:?} was not gathered yet and has children refering to it, continuing traversal through those children..."); + } else { + warn!("Incoherent DAG at: {current_addr:?}"); + return Err(DagError::IncoherentDag( + *current_addr, + format!("descendants via DAG: {descendants_via_dag:?} do not match descendants via TX: {descendants_via_tx:?}") + )); + } } // continue traversal @@ -430,6 +454,7 @@ impl SpendDag { pub fn record_faults(&mut self, source: &SpendAddress) -> Result<(), DagError> { let faults = self.verify(source)?; + self.faults.clear(); for f in faults { self.faults.entry(f.spend_address()).or_default().insert(f); } @@ -520,9 +545,14 @@ impl SpendDag { // get the ancestors of this spend let ancestor_spends = match self.get_ancestor_spends(spend) { Ok(a) => a, - Err(e) => { - debug!("Failed to get ancestor spends of: {addr:?} {e}"); - recorded_faults.insert(e); + Err(fault) => { + debug!("Failed to get ancestor spends of {addr:?}: {fault}"); + recorded_faults.insert(fault.clone()); + + // if ancestry is invalid, poison all the descendants + let poison = format!("ancestry issue: {fault}"); + let descendants_faults = self.poison_all_descendants(spend, poison)?; + recorded_faults.extend(descendants_faults); return Ok(recorded_faults); } }; diff --git a/sn_client/src/audit/spend_dag_building.rs b/sn_client/src/audit/spend_dag_building.rs index c81c53e0eb..0a269b62c2 100644 --- a/sn_client/src/audit/spend_dag_building.rs +++ b/sn_client/src/audit/spend_dag_building.rs @@ -7,7 +7,7 @@ // permissions and limitations relating to use of the SAFE Network Software. use super::{Client, SpendDag}; -use crate::{Error, Result}; +use crate::Error; use futures::{future::join_all, StreamExt}; use sn_networking::{GetRecordError, NetworkError}; @@ -93,7 +93,7 @@ impl Client { info!("Reached UTXO at {addr:?}"); } Err(err) => { - error!("Could not verify transfer at {addr:?}: {err:?}"); + error!("Failed to get spend at {addr:?} during DAG collection: {err:?}"); } } } @@ -175,7 +175,7 @@ impl Client { let parent_tx_hash = parent_tx.hash(); let parent_keys = parent_tx.inputs.iter().map(|input| input.unique_pubkey); let addrs_to_verify = parent_keys.map(|k| SpendAddress::from_unique_pubkey(&k)); - debug!("Depth {depth} - checking parent Tx : {parent_tx_hash:?}"); + debug!("Depth {depth} - checking parent Tx : {parent_tx_hash:?} with inputs: {addrs_to_verify:?}"); // get all parent spends in parallel let tasks: Vec<_> = addrs_to_verify @@ -184,8 +184,10 @@ impl Client { .collect(); let spends = join_all(tasks).await .into_iter() - .collect::>>() - .map_err(|err| WalletError::FailedToGetSpend(format!("at depth {depth} - Failed to get spends from network for parent Tx {parent_tx_hash:?}: {err}")))?; + .zip(addrs_to_verify.clone()) + .map(|(maybe_spend, a)| + maybe_spend.map_err(|err| WalletError::CouldNotVerifyTransfer(format!("at depth {depth} - Failed to get spend {a:?} from network for parent Tx {parent_tx_hash:?}: {err}")))) + .collect::>>()?; debug!( "Depth {depth} - Got {:?} spends for parent Tx: {parent_tx_hash:?}", spends.len() @@ -259,7 +261,8 @@ impl Client { dag: &mut SpendDag, max_depth: Option, ) -> WalletResult<()> { - info!("Gathering spend DAG from utxos..."); + let main_dag_src = dag.source(); + info!("Expanding spend DAG with source: {main_dag_src:?} from utxos..."); let utxos = dag.get_utxos(); let mut stream = futures::stream::iter(utxos.into_iter()) @@ -271,9 +274,14 @@ impl Client { while let Some((res, addr)) = stream.next().await { match res { - Ok(d) => dag.merge(d), + Ok(sub_dag) => { + debug!("Gathered sub DAG from: {addr:?}"); + if let Err(e) = dag.merge(sub_dag) { + warn!("Failed to merge sub dag from {addr:?} into dag: {e}"); + } + } Err(e) => warn!("Failed to gather sub dag from {addr:?}: {e}"), - } + }; } dag.record_faults(&dag.source()) From 3233f69c2399efce6a1b8f99256a50db9fb6d6e6 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Fri, 5 Apr 2024 17:05:29 +0100 Subject: [PATCH 018/205] ci: ensure cross uses version-mode variable The ARM builds are done using a tool called `cross`, which uses Docker containers, and therefore the compile-time variables need to be passed to the containers. This is done using the `CROSS_CONTAINER_OPTS` variable. I tested this locally to confirm the desired effect. --- .github/workflows/release.yml | 3 --- Justfile | 6 ++++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 46bd8631b2..f664b825d9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -59,9 +59,6 @@ jobs: - name: build release artifacts shell: bash run: | - # make sure the version env variable is set - export NETWORK_VERSION_MODE=${{ env.NETWORK_VERSION_MODE }} - echo "NETWORK_VERSION_MODE=$NETWORK_VERSION_MODE" just build-release-artifacts "${{ matrix.target }}" - uses: actions/upload-artifact@main diff --git a/Justfile b/Justfile index 6464d80229..49bd390fe8 100644 --- a/Justfile +++ b/Justfile @@ -104,6 +104,12 @@ build-release-artifacts arch: rm -rf artifacts mkdir artifacts cargo clean + + if [[ -n "${NETWORK_VERSION_MODE+x}"]]; then + echo "The NETWORK_VERSION_MODE variable is set to $NETWORK_VERSION_MODE" + export CROSS_CONTAINER_OPTS="--env NETWORK_VERSION_MODE=$NETWORK_VERSION_MODE" + fi + if [[ $arch == arm* || $arch == armv7* || $arch == aarch64* ]]; then cargo install cross cross build --release --features="network-contacts,distribution" --target $arch --bin safe From a863bed8f4f5b3ff799cc0e03a95b0a767d5b15a Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 3 Apr 2024 19:43:53 +0530 Subject: [PATCH 019/205] chore(ci): edit build info to resolve release issue --- sn_build_info/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sn_build_info/src/lib.rs b/sn_build_info/src/lib.rs index 82e690eb5a..038826d6cf 100644 --- a/sn_build_info/src/lib.rs +++ b/sn_build_info/src/lib.rs @@ -22,7 +22,7 @@ pub const fn git_describe() -> &'static str { env!("VERGEN_GIT_DESCRIBE") } -/// Current git branch. +/// The current git branch. pub const fn git_branch() -> &'static str { env!("VERGEN_GIT_BRANCH") } From c51f7dd8887926895d1a0d0270e3032eabb67f25 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 3 Apr 2024 16:42:06 +0100 Subject: [PATCH 020/205] chore: output the date in the version This change exists just to generate a new version number. We can remove it if necessary. --- sn_build_info/build.rs | 1 + sn_build_info/src/lib.rs | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/sn_build_info/build.rs b/sn_build_info/build.rs index 8ef6278521..392c55da4e 100644 --- a/sn_build_info/build.rs +++ b/sn_build_info/build.rs @@ -9,6 +9,7 @@ use vergen::EmitBuilder; fn main() -> Result<(), Box> { EmitBuilder::builder() + .build_date() // Emit the short SHA-1 hash of the current commit .git_sha(true) // Emit the current branch name diff --git a/sn_build_info/src/lib.rs b/sn_build_info/src/lib.rs index 038826d6cf..6b858254ac 100644 --- a/sn_build_info/src/lib.rs +++ b/sn_build_info/src/lib.rs @@ -13,7 +13,9 @@ pub const fn git_info() -> &'static str { " / ", env!("VERGEN_GIT_BRANCH"), " / ", - env!("VERGEN_GIT_DESCRIBE") + env!("VERGEN_GIT_DESCRIBE"), + " / ", + env!("VERGEN_BUILD_DATE") ) } From 0bfc23333b8ce86d5d0db297dc4278c7ee9f457a Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 3 Apr 2024 17:00:58 +0100 Subject: [PATCH 021/205] chore: clarifications to version bumping script There was an issue with the script always applying 0 for the pre-release identifier. We got into a situation where it actually needed to be incremented by 1, because there was already a crate published at the 0 version. Added some text output to help clarify what the script is doing. Also tidied the style by consistently applying Bash syntax (as opposed to POSIX compatible) and a tab size of 2. --- resources/scripts/bump_version.sh | 70 +++++++++++++++---------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/resources/scripts/bump_version.sh b/resources/scripts/bump_version.sh index 6503d3a263..748f2f3019 100755 --- a/resources/scripts/bump_version.sh +++ b/resources/scripts/bump_version.sh @@ -2,28 +2,26 @@ set -e - # Suffix to append to the version. Passed as an argument to this script. SUFFIX="$1" # Ensure cargo set-version is installed if ! cargo set-version --help > /dev/null 2>&1; then - echo "cargo set-version command not found." - echo "Please install cargo-edit with the command: cargo install cargo-edit --features vendored-openssl" - exit 1 + echo "cargo set-version command not found." + echo "Please install cargo-edit with the command: cargo install cargo-edit --features vendored-openssl" + exit 1 fi # Ensure the suffix is either alpha or beta -if [ -n "$SUFFIX" ]; then - if [[ "$SUFFIX" != "alpha" ]] && [[ "$SUFFIX" != "beta" ]]; then - echo "Invalid suffix. Suffix must be either 'alpha' or 'beta'." - exit 1 - fi +if [[ -n "$SUFFIX" ]]; then + if [[ "$SUFFIX" != "alpha" ]] && [[ "$SUFFIX" != "beta" ]]; then + echo "Invalid suffix. Suffix must be either 'alpha' or 'beta'." + exit 1 + fi fi release-plz update 2>&1 | tee bump_version_output - crates_bumped=() while IFS= read -r line; do name=$(echo "$line" | awk -F"\`" '{print $2}') @@ -37,38 +35,40 @@ if [[ $len -eq 0 ]]; then exit 0 fi -# remove any performed changes if we're applying a suffix -if [ -n "$SUFFIX" ]; then - git checkout -- . +if [[ -n "$SUFFIX" ]]; then + echo "We are releasing to the $SUFFIX channel" + echo "Versions with $SUFFIX are not supported by release-plz" + echo "Reverting changes by release-plz" + git checkout -- . fi commit_message="chore(release): " for crate in "${crates_bumped[@]}"; do - # Extract the crate name and version in a cross-platform way - crate_name=$(echo "$crate" | sed -E 's/-v.*$//') - version=$(echo "$crate" | sed -E 's/^.*-v(.*)$/\1/') - new_version=$version - echo "the crate is: $crate_name" - # if we're changing the release channel... - if [ -n "$SUFFIX" ]; then - #if we're already in a realse channel, reapplying the suffix will reset things. - if [[ "$version" == *"-alpha."* || "$version" == *"-beta."* ]]; then - #remove any existing channel + version - base_version=$(echo "$version" | sed -E 's/(-alpha\.[0-9]+|-beta\.[0-9]+)$//') - new_version="${base_version}-${SUFFIX}.0" - else - new_version="${version}-${SUFFIX}.0" - fi + # Extract the crate name and version in a cross-platform way + crate_name=$(echo "$crate" | sed -E 's/-v.*$//') + version=$(echo "$crate" | sed -E 's/^.*-v(.*)$/\1/') + new_version=$version + + echo "----------------------------------------------------------" + echo "Processing $crate_name" + echo "----------------------------------------------------------" + if [[ -n "$SUFFIX" ]]; then + # if we're already in a release channel, reapplying the suffix will reset things. + if [[ "$version" == *"-alpha."* || "$version" == *"-beta."* ]]; then + base_version=$(echo "$version" | sed -E 's/(-alpha\.[0-9]+|-beta\.[0-9]+)$//') + pre_release_identifier=$(echo "$version" | sed -E 's/.*-(alpha|beta)\.([0-9]+)$/\2/') + new_version="${base_version}-${SUFFIX}.$pre_release_identifier" else - # For main release, strip any alpha or beta suffix from the version - new_version=$(echo "$version" | sed -E 's/(-alpha\.[0-9]+|-beta\.[0-9]+)$//') + new_version="${version}-${SUFFIX}.0" fi + else + # For main release, strip any alpha or beta suffix from the version + new_version=$(echo "$version" | sed -E 's/(-alpha\.[0-9]+|-beta\.[0-9]+)$//') + fi - # set the version - crate=$new_version - cargo set-version -p $crate_name $new_version - # update the commit msg - commit_message="${commit_message}${crate_name}-v$new_version/" + echo "Using set-version to apply $new_version to $crate_name" + cargo set-version -p $crate_name $new_version + commit_message="${commit_message}${crate_name}-v$new_version/" # append crate to commit message done commit_message=${commit_message%/} # strip off trailing '/' character From cb76aa254b32329ca9efc19f6eb10596a716caf0 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 3 Apr 2024 19:07:36 +0100 Subject: [PATCH 022/205] docs: clarify client::new description Just removed a few words to try and effect another change in `sn_client`, to get a new version. This crate has already been published and references an older version of `sn_protocol`, which is causing a problem because `sn_cli` is referencing a newer version of it. --- Cargo.lock | 145 +++++++++++++++++++++---------------------- sn_client/src/api.rs | 2 +- 2 files changed, 73 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a7e77508a4..8540c35f25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -339,7 +339,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -350,7 +350,7 @@ checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -519,9 +519,9 @@ checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" [[package]] name = "bitcoin" -version = "0.31.1" +version = "0.31.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd00f3c09b5f21fb357abe32d29946eb8bb7a0862bae62c0b5e4a692acbbe73c" +checksum = "6c85783c2fe40083ea54a33aa2f0ba58831d90fcd190f5bdc47e74e84d2a96ae" dependencies = [ "base64 0.21.7", "bech32", @@ -834,9 +834,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.35" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaf5903dcbc0a39312feb77df2ff4c76387d591b9fc7b04a238dcf8bb62639a" +checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e" dependencies = [ "android-tzdata", "iana-time-zone", @@ -918,7 +918,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim 0.11.0", + "strsim 0.11.1", ] [[package]] @@ -930,7 +930,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -1211,7 +1211,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -1233,7 +1233,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", "synstructure 0.13.1", ] @@ -1258,7 +1258,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.10.0", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -1269,7 +1269,7 @@ checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ "darling_core", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -1309,9 +1309,9 @@ dependencies = [ [[package]] name = "der" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" dependencies = [ "const-oid", "zeroize", @@ -1428,7 +1428,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -1519,7 +1519,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -1789,7 +1789,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -1908,7 +1908,7 @@ dependencies = [ "bstr", "log", "regex-automata 0.4.6", - "regex-syntax 0.8.2", + "regex-syntax 0.8.3", ] [[package]] @@ -1965,9 +1965,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fbd2820c5e49886948654ab546d0688ff24530286bdcf8fca3cefb16d4618eb" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ "bytes", "fnv", @@ -2995,7 +2995,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -3105,13 +3105,12 @@ dependencies = [ [[package]] name = "libredox" -version = "0.0.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.5.0", "libc", - "redox_syscall", ] [[package]] @@ -3189,9 +3188,9 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "memmap2" @@ -3235,9 +3234,9 @@ dependencies = [ [[package]] name = "minreq" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb3371dfc7b772c540da1380123674a8e20583aca99907087d990ca58cf44203" +checksum = "00a000cf8bbbfb123a9bdc66b61c2885a4bb038df4f2629884caafabeb76b0f9" dependencies = [ "log", "once_cell", @@ -3308,7 +3307,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -3444,9 +3443,9 @@ dependencies = [ [[package]] name = "netlink-sys" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6471bf08e7ac0135876a9581bf3217ef0333c191c128d34878079f42ee150411" +checksum = "416060d346fbaf1f23f9512963e3e878f1a78e707cb699ba9215761754244307" dependencies = [ "bytes", "futures", @@ -3880,9 +3879,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.8" +version = "2.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f8023d0fb78c8e03784ea1c7f3fa36e68a723138990b8d5a47d916b651e7a8" +checksum = "311fb059dee1a7b802f036316d790138c613a4e8b180c822e3925a662e9f0c95" dependencies = [ "memchr", "thiserror", @@ -3891,9 +3890,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.8" +version = "2.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0d24f72393fd16ab6ac5738bc33cdb6a9aa73f8b902e8fe29cf4e67d7dd1026" +checksum = "f73541b156d32197eecda1a4014d7f868fd2bcb3c550d5386087cfba442bf69c" dependencies = [ "pest", "pest_generator", @@ -3901,22 +3900,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.8" +version = "2.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc17e2a6c7d0a492f0158d7a4bd66cc17280308bbaff78d5bef566dca35ab80" +checksum = "c35eeed0a3fab112f75165fdc026b3913f4183133f19b49be773ac9ea966e8bd" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] name = "pest_meta" -version = "2.7.8" +version = "2.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "934cd7631c050f4674352a6e835d5f6711ffbfb9345c2fc0107155ac495ae293" +checksum = "2adbf29bb9776f28caece835398781ab24435585fe0d4dc1374a61db5accedca" dependencies = [ "once_cell", "pest", @@ -3952,14 +3951,14 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -3985,9 +3984,9 @@ checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "platforms" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626dec3cac7cc0e1577a2ec3fc496277ec2baa084bebad95bb6fdbfae235f84c" +checksum = "db23d408679286588f4d4644f965003d056e3dd5abcaaa938116871d7ce2fee7" [[package]] name = "plist" @@ -4205,7 +4204,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -4222,7 +4221,7 @@ dependencies = [ "rand", "rand_chacha", "rand_xorshift", - "regex-syntax 0.8.2", + "regex-syntax 0.8.3", "rusty-fork", "tempfile", "unarray", @@ -4506,9 +4505,9 @@ dependencies = [ [[package]] name = "redox_users" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" +checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" dependencies = [ "getrandom", "libredox", @@ -4524,7 +4523,7 @@ dependencies = [ "aho-corasick", "memchr", "regex-automata 0.4.6", - "regex-syntax 0.8.2", + "regex-syntax 0.8.3", ] [[package]] @@ -4544,7 +4543,7 @@ checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.2", + "regex-syntax 0.8.3", ] [[package]] @@ -4555,9 +4554,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "reqwest" @@ -5022,7 +5021,7 @@ checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -5732,9 +5731,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "strsim" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" @@ -5755,7 +5754,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -5800,9 +5799,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.55" +version = "2.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "002a1b3dbf967edfafc32655d0f377ab0bb7b994aa1d32c8cc7e9b8bf3ebb8f0" +checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" dependencies = [ "proc-macro2", "quote", @@ -5835,7 +5834,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -5937,7 +5936,7 @@ checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -6055,9 +6054,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.36.0" +version = "1.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" dependencies = [ "backtrace", "bytes", @@ -6090,7 +6089,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -6312,7 +6311,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -6755,7 +6754,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", "wasm-bindgen-shared", ] @@ -6789,7 +6788,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -7139,9 +7138,9 @@ dependencies = [ [[package]] name = "xml-rs" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcb9cbac069e033553e8bb871be2fbdffcab578eb25bd0f7c508cedc6dcd75a" +checksum = "791978798f0597cfc70478424c2b4fdc2b7a8024aaff78497ef00f24ef674193" [[package]] name = "xmltree" @@ -7223,7 +7222,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -7243,7 +7242,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.58", ] [[package]] @@ -7287,9 +7286,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.9+zstd.1.5.5" +version = "2.0.10+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" dependencies = [ "cc", "pkg-config", diff --git a/sn_client/src/api.rs b/sn_client/src/api.rs index 35ded6f5ad..9fb21d5f63 100644 --- a/sn_client/src/api.rs +++ b/sn_client/src/api.rs @@ -64,7 +64,7 @@ impl Client { /// Instantiate a new client. /// - /// Optionally specify the maximum time the client will wait for a connection to the network before timing out. + /// Optionally specify the maximum time the client will wait for a connection before timing out. /// Defaults to 180 seconds. /// /// # Arguments From 54223b414d6334ff99ca635fede0b948cba9ae69 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 3 Apr 2024 23:22:06 +0100 Subject: [PATCH 023/205] docs: clarify client documentation Again, these changes are just to effect a release. A change to the release workflow also removes testing the environment variable and re-exporting it. On the last build I ran, it appears to have worked as expected, where previously it didn't. Doesn't really make any sense. --- sn_client/src/api.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sn_client/src/api.rs b/sn_client/src/api.rs index 9fb21d5f63..7fad35d576 100644 --- a/sn_client/src/api.rs +++ b/sn_client/src/api.rs @@ -57,14 +57,15 @@ const CONNECTION_TIMEOUT: Duration = Duration::from_secs(30); const INACTIVITY_TIMEOUT: Duration = Duration::from_secs(30); impl Client { - /// A quick client that only takes some peers to connect to + /// A quick client with a random secret key and some peers. pub async fn quick_start(peers: Option>) -> Result { Self::new(SecretKey::random(), peers, None, None).await } /// Instantiate a new client. /// - /// Optionally specify the maximum time the client will wait for a connection before timing out. + /// Optionally specify the duration for the connection timeout. + /// /// Defaults to 180 seconds. /// /// # Arguments From 71ea622c1bcbed9917b778440d271eb4d6024f4e Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 4 Apr 2024 10:39:53 +0900 Subject: [PATCH 024/205] fix(protocol): evaluate NETWORK_VERSION_MODE at compile time --- sn_peers_acquisition/src/lib.rs | 2 +- sn_protocol/src/version.rs | 18 ++++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/sn_peers_acquisition/src/lib.rs b/sn_peers_acquisition/src/lib.rs index 26e192a18c..e1698ea374 100644 --- a/sn_peers_acquisition/src/lib.rs +++ b/sn_peers_acquisition/src/lib.rs @@ -25,7 +25,7 @@ lazy_static! { // URL containing the multi-addresses of the bootstrap nodes. pub static ref NETWORK_CONTACTS_URL: String = { let version = get_network_version(); - let version_prefix = if !version.is_empty() { format!("{version}-") } else { version }; + let version_prefix = if !version.is_empty() { format!("{version}-") } else { version.to_string() }; format!("https://sn-testnet.s3.eu-west-2.amazonaws.com/{version_prefix}network-contacts") }; } diff --git a/sn_protocol/src/version.rs b/sn_protocol/src/version.rs index abc959cdca..3c5074c1cc 100644 --- a/sn_protocol/src/version.rs +++ b/sn_protocol/src/version.rs @@ -8,10 +8,6 @@ use lazy_static::lazy_static; -/// Set this env variable to provide custom network versioning. If it is set to 'restricted', then the git branch name -/// is used as the version string. Else we directly use the passed in string as the version. -const NETWORK_VERSION_MODE_ENV_VARIABLE: &str = "NETWORK_VERSION_MODE"; - lazy_static! { /// The node version used during Identify Behaviour. pub static ref IDENTIFY_NODE_VERSION_STR: String = @@ -53,16 +49,18 @@ lazy_static! { /// If the network version mode env variable is set to `restricted`, then the git branch is used as the version. /// Else any non empty string is used as the version string. /// If the env variable is empty or not set, then we do not apply any network versioning. -pub fn get_network_version() -> String { - match std::env::var(NETWORK_VERSION_MODE_ENV_VARIABLE) { - Ok(value) if !value.is_empty() => { +pub fn get_network_version() -> &'static str { + // Set this env variable to provide custom network versioning. If it is set to 'restricted', then the git branch name + // is used as the version string. Else we directly use the passed in string as the version. + match option_env!("NETWORK_VERSION_MODE") { + Some(value) => { if value == "restricted" { - sn_build_info::git_branch().to_string() + sn_build_info::git_branch() } else { value } } - _ => "".to_string(), + _ => "", } } @@ -70,7 +68,7 @@ pub fn get_network_version() -> String { fn write_network_version_with_slash() -> String { let version = get_network_version(); if version.is_empty() { - version + version.to_string() } else { format!("/{version}") } From f50a71f556fe4abe7313a608c2883ecf5fefdf9d Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Sat, 6 Apr 2024 00:50:37 +0530 Subject: [PATCH 025/205] feat: report protocol mismatch error --- sn_cli/src/bin/main.rs | 23 ++++++++++------ sn_client/src/api.rs | 30 ++++++++++++++++++--- sn_client/src/error.rs | 3 +++ sn_client/src/event.rs | 5 ++++ sn_faucet/src/main.rs | 32 ++++++++++++---------- sn_networking/src/driver.rs | 1 + sn_networking/src/event.rs | 53 ++++++++++++++++++++++++++----------- sn_node/src/node.rs | 3 +++ 8 files changed, 109 insertions(+), 41 deletions(-) diff --git a/sn_cli/src/bin/main.rs b/sn_cli/src/bin/main.rs index 13ce887081..67294df045 100644 --- a/sn_cli/src/bin/main.rs +++ b/sn_cli/src/bin/main.rs @@ -112,7 +112,8 @@ async fn main() -> Result<()> { // get the broadcaster as we want to have our own progress bar. let broadcaster = ClientEventsBroadcaster::default(); - let progress_bar_handler = spawn_connection_progress_bar(broadcaster.subscribe()); + let (progress_bar, progress_bar_handler) = + spawn_connection_progress_bar(broadcaster.subscribe()); let result = Client::new( secret_key, @@ -121,11 +122,15 @@ async fn main() -> Result<()> { Some(broadcaster), ) .await; - - // await on the progress bar to complete before handling the client result. If client errors out, we would - // want to make the progress bar clean up gracefully. + let client = match result { + Ok(client) => client, + Err(err) => { + // clean up progress bar + progress_bar.finish_with_message("Could not connect to the network"); + return Err(err.into()); + } + }; progress_bar_handler.await?; - let client = result?; // default to verifying storage let should_verify_store = !opt.no_verify; @@ -153,9 +158,10 @@ async fn main() -> Result<()> { /// Helper to subscribe to the client events broadcaster and spin up a progress bar that terminates when the /// client successfully connects to the network or if it errors out. -fn spawn_connection_progress_bar(mut rx: ClientEventsReceiver) -> JoinHandle<()> { +fn spawn_connection_progress_bar(mut rx: ClientEventsReceiver) -> (ProgressBar, JoinHandle<()>) { // Network connection progress bar let progress_bar = ProgressBar::new_spinner(); + let progress_bar_clone = progress_bar.clone(); progress_bar.enable_steady_tick(Duration::from_millis(120)); progress_bar.set_message("Connecting to The SAFE Network..."); let new_style = progress_bar.style().tick_chars("⠁⠂⠄⡀⢀⠠⠐⠈🔗"); @@ -163,7 +169,7 @@ fn spawn_connection_progress_bar(mut rx: ClientEventsReceiver) -> JoinHandle<()> progress_bar.set_message("Connecting to The SAFE Network..."); - tokio::spawn(async move { + let handle = tokio::spawn(async move { let mut peers_connected = 0; loop { match rx.recv().await { @@ -190,7 +196,8 @@ fn spawn_connection_progress_bar(mut rx: ClientEventsReceiver) -> JoinHandle<()> _ => {} } } - }) + }); + (progress_bar_clone, handle) } fn get_client_secret_key(root_dir: &PathBuf) -> Result { diff --git a/sn_client/src/api.rs b/sn_client/src/api.rs index 7fad35d576..626c3c389f 100644 --- a/sn_client/src/api.rs +++ b/sn_client/src/api.rs @@ -191,7 +191,9 @@ impl Client { // loop to connect to the network let mut is_connected = false; let connection_timeout = connection_timeout.unwrap_or(CONNECTION_TIMEOUT); + let mut unsupported_protocol_tracker: Option<(String, String)> = None; + debug!("Client connection timeout: {connection_timeout:?}"); let mut connection_timeout_interval = interval(connection_timeout); // first tick completes immediately connection_timeout_interval.tick().await; @@ -200,16 +202,26 @@ impl Client { tokio::select! { _ = connection_timeout_interval.tick() => { if !is_connected { + if let Some((our_protocol, their_protocols)) = unsupported_protocol_tracker { + error!("Timeout: Client could not connect to the network as it does not support the protocol"); + break Err(Error::UnsupportedProtocol(our_protocol, their_protocols)); + } error!("Timeout: Client failed to connect to the network within {connection_timeout:?}"); - return Err(Error::ConnectionTimeout(connection_timeout)); + break Err(Error::ConnectionTimeout(connection_timeout)); } } event = client_events_rx.recv() => { match event { + // we do not error out directly as we might still connect if the other initial peers are from + // the correct network. + Ok(ClientEvent::PeerWithUnsupportedProtocol { our_protocol, their_protocol }) => { + warn!(%our_protocol, %their_protocol, "Client tried to connect to a peer with an unsupported protocol. Tracking the latest one"); + unsupported_protocol_tracker = Some((our_protocol, their_protocol)); + } Ok(ClientEvent::ConnectedToNetwork) => { is_connected = true; info!("Client connected to the Network {is_connected:?}."); - break; + break Ok(()); } Ok(ClientEvent::InactiveClient(timeout)) => { if is_connected { @@ -221,12 +233,12 @@ impl Client { Err(err) => { error!("Unexpected error during client startup {err:?}"); println!("Unexpected error during client startup {err:?}"); - return Err(err.into()); + break Err(err.into()); } _ => {} } }} - } + }?; Ok(client) } @@ -252,6 +264,16 @@ impl Client { debug!("{peers_added}/{CLOSE_GROUP_SIZE} initial peers found.",); } } + NetworkEvent::PeerWithUnsupportedProtocol { + our_protocol, + their_protocol, + } => { + self.events_broadcaster + .broadcast(ClientEvent::PeerWithUnsupportedProtocol { + our_protocol, + their_protocol, + }); + } _other => {} } diff --git a/sn_client/src/error.rs b/sn_client/src/error.rs index a5880bd420..13bf844ed1 100644 --- a/sn_client/src/error.rs +++ b/sn_client/src/error.rs @@ -112,6 +112,9 @@ pub enum Error { #[error("Could not find register after batch sync: {0:?}")] RegisterNotFoundAfterUpload(XorName), + #[error("Could not connect due to incompatible network protocols. Our protocol: {0} Network protocol: {1}")] + UnsupportedProtocol(String, String), + // ------ Upload Errors -------- #[error("Overflow occurred while adding values")] NumericOverflow, diff --git a/sn_client/src/event.rs b/sn_client/src/event.rs index 0f53c6a79f..25b18eef1c 100644 --- a/sn_client/src/event.rs +++ b/sn_client/src/event.rs @@ -42,6 +42,11 @@ pub enum ClientEvent { /// A peer has been added to the Routing table. /// Also contains the max number of peers to connect to before we receive ClientEvent::ConnectedToNetwork PeerAdded { max_peers_to_connect: usize }, + /// We've encountered a Peer with an unsupported protocol. + PeerWithUnsupportedProtocol { + our_protocol: String, + their_protocol: String, + }, /// The client has been connected to the network ConnectedToNetwork, /// No network activity has been received for a given duration diff --git a/sn_faucet/src/main.rs b/sn_faucet/src/main.rs index f35bb810b0..935d0066f2 100644 --- a/sn_faucet/src/main.rs +++ b/sn_faucet/src/main.rs @@ -60,19 +60,21 @@ async fn main() -> Result<()> { let secret_key = bls::SecretKey::random(); let broadcaster = ClientEventsBroadcaster::default(); - let handle = spawn_connection_progress_bar(broadcaster.subscribe()); + let (progress_bar, handle) = spawn_connection_progress_bar(broadcaster.subscribe()); let result = Client::new(secret_key, bootstrap_peers, None, Some(broadcaster)).await; - - // await on the progress bar to complete before handling the client result. If client errors out, we would - // want to make the progress bar clean up gracefully. - handle.await?; - match result { - Ok(client) => { - if let Err(err) = faucet_cmds(opt.cmd.clone(), &client).await { - error!("Failed to run faucet cmd {:?} with err {err:?}", opt.cmd) - } + let client = match result { + Ok(client) => client, + Err(err) => { + // clean up progress bar + progress_bar.finish_with_message("Could not connect to the network"); + error!("Failed to get Client with err {err:?}"); + return Err(err.into()); } - Err(err) => error!("Failed to get Client with err {err:?}"), + }; + handle.await?; + + if let Err(err) = faucet_cmds(opt.cmd.clone(), &client).await { + error!("Failed to run faucet cmd {:?} with err {err:?}", opt.cmd) } Ok(()) @@ -80,9 +82,10 @@ async fn main() -> Result<()> { /// Helper to subscribe to the client events broadcaster and spin up a progress bar that terminates when the /// client successfully connects to the network or if it errors out. -fn spawn_connection_progress_bar(mut rx: ClientEventsReceiver) -> JoinHandle<()> { +fn spawn_connection_progress_bar(mut rx: ClientEventsReceiver) -> (ProgressBar, JoinHandle<()>) { // Network connection progress bar let progress_bar = ProgressBar::new_spinner(); + let progress_bar_clone = progress_bar.clone(); progress_bar.enable_steady_tick(Duration::from_millis(120)); progress_bar.set_message("Connecting to The SAFE Network..."); let new_style = progress_bar.style().tick_chars("⠁⠂⠄⡀⢀⠠⠐⠈🔗"); @@ -90,7 +93,7 @@ fn spawn_connection_progress_bar(mut rx: ClientEventsReceiver) -> JoinHandle<()> progress_bar.set_message("Connecting to The SAFE Network..."); - tokio::spawn(async move { + let handle = tokio::spawn(async move { let mut peers_connected = 0; loop { match rx.recv().await { @@ -117,7 +120,8 @@ fn spawn_connection_progress_bar(mut rx: ClientEventsReceiver) -> JoinHandle<()> _ => {} } } - }) + }); + (progress_bar_clone, handle) } #[derive(Parser)] diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index 42d5a2c5cd..b91ddfa112 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -328,6 +328,7 @@ impl NetworkBuilder { // 1mb packet size let _ = kad_cfg + .set_kbucket_inserts(libp2p::kad::BucketInserts::Manual) .set_max_packet_size(MAX_PACKET_SIZE) // Require iterative queries to use disjoint paths for increased resiliency in the presence of potentially adversarial nodes. .disjoint_query_paths(true) diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index c61e2b0e13..45c4d95b7f 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -30,7 +30,7 @@ use libp2p::{ Multiaddr, PeerId, TransportError, }; use rand::{rngs::OsRng, Rng}; -use sn_protocol::version::IDENTIFY_NODE_VERSION_STR; +use sn_protocol::version::{IDENTIFY_NODE_VERSION_STR, IDENTIFY_PROTOCOL_STR}; use sn_protocol::{ get_port_from_multiaddr, messages::{CmdResponse, Query, Request, Response}, @@ -107,8 +107,13 @@ pub enum NetworkEvent { }, /// Peer has been added to the Routing Table. And the number of connected peers. PeerAdded(PeerId, usize), - // Peer has been removed from the Routing Table. And the number of connected peers. + /// Peer has been removed from the Routing Table. And the number of connected peers. PeerRemoved(PeerId, usize), + /// The peer does not support our protocol + PeerWithUnsupportedProtocol { + our_protocol: String, + their_protocol: String, + }, /// The records bearing these keys are to be fetched from the holder or the network KeysToFetchForReplication(Vec<(PeerId, RecordKey)>), /// Started listening on a new address @@ -120,13 +125,9 @@ pub enum NetworkEvent { /// List of peer nodes that failed to fetch replication copy from. FailedToFetchHolders(BTreeSet), /// A peer in RT that supposed to be verified. - BadNodeVerification { - peer_id: PeerId, - }, + BadNodeVerification { peer_id: PeerId }, /// Quotes to be verified - QuoteVerification { - quotes: Vec<(PeerId, PaymentQuote)>, - }, + QuoteVerification { quotes: Vec<(PeerId, PaymentQuote)> }, /// Carry out chunk proof check against the specified record and peer ChunkProofVerification { peer_id: PeerId, @@ -153,6 +154,12 @@ impl Debug for NetworkEvent { "NetworkEvent::PeerRemoved({peer_id:?}, {connected_peers})" ) } + NetworkEvent::PeerWithUnsupportedProtocol { + our_protocol, + their_protocol, + } => { + write!(f, "NetworkEvent::PeerWithUnsupportedProtocol({our_protocol:?}, {their_protocol:?})") + } NetworkEvent::KeysToFetchForReplication(list) => { let keys_len = list.len(); write!(f, "NetworkEvent::KeysForReplication({keys_len:?})") @@ -219,10 +226,20 @@ impl SwarmDriver { libp2p::identify::Event::Received { peer_id, info } => { trace!(%peer_id, ?info, "identify: received info"); + if info.protocol_version != IDENTIFY_PROTOCOL_STR.to_string() { + warn!(?info.protocol_version, "identify: {peer_id:?} does not have the same protocol. Our IDENTIFY_PROTOCOL_STR: {:?}", IDENTIFY_PROTOCOL_STR.as_str()); + + self.send_event(NetworkEvent::PeerWithUnsupportedProtocol { + our_protocol: IDENTIFY_PROTOCOL_STR.to_string(), + their_protocol: info.protocol_version, + }); + + return Ok(()); + } + let has_dialed = self.dialed_peers.contains(&peer_id); - let peer_is_agent = info - .agent_version - .starts_with(&IDENTIFY_NODE_VERSION_STR.to_string()); + let peer_is_node = + info.agent_version == IDENTIFY_NODE_VERSION_STR.to_string(); // If we're not in local mode, only add globally reachable addresses. // Strip the `/p2p/...` part of the multiaddresses. @@ -243,8 +260,8 @@ impl SwarmDriver { // When received an identify from un-dialed peer, try to dial it // The dial shall trigger the same identify to be sent again and confirm - // peer is external accessable, hence safe to be added into RT. - if !self.local && peer_is_agent && !has_dialed { + // peer is external accessible, hence safe to be added into RT. + if !self.local && peer_is_node && !has_dialed { // Only need to dial back for not fulfilled kbucket let (kbucket_full, ilog2) = if let Some(kbucket) = self.swarm.behaviour_mut().kademlia.kbucket(peer_id) @@ -275,7 +292,7 @@ impl SwarmDriver { }; if !kbucket_full { - info!(%peer_id, ?addrs, "received identify info from undialed peer for not full kbucket {:?}, dail back to confirm external accesable", ilog2); + info!(%peer_id, ?addrs, "received identify info from undialed peer for not full kbucket {ilog2:?}, dial back to confirm external accessible"); self.dialed_peers .push(peer_id) .map_err(|_| NetworkError::CircularVecPopFrontError)?; @@ -297,7 +314,7 @@ impl SwarmDriver { } // If we are not local, we care only for peers that we dialed and thus are reachable. - if self.local || has_dialed && peer_is_agent { + if self.local || has_dialed && peer_is_node { // To reduce the bad_node check resource usage, // during the connection establish process, only check cached black_list // The periodical check, which involves network queries shall filter @@ -1029,6 +1046,12 @@ impl SwarmDriver { event_string = "kad_event::UnroutablePeer"; trace!(peer_id = %peer, "kad::Event: UnroutablePeer"); } + kad::Event::RoutablePeer { peer, .. } => { + // We get this when we don't add a peer via the identify step. + // And we don't want to add these as they were rejected by identify for some reason. + event_string = "kad_event::RoutablePeer"; + trace!(peer_id = %peer, "kad::Event: RoutablePeer"); + } other => { event_string = "kad_event::Other"; trace!("kad::Event ignored: {other:?}"); diff --git a/sn_node/src/node.rs b/sn_node/src/node.rs index 8dd1314cfe..0ee4b8ffe2 100644 --- a/sn_node/src/node.rs +++ b/sn_node/src/node.rs @@ -326,6 +326,9 @@ impl Node { Self::try_interval_replication(net); }); } + NetworkEvent::PeerWithUnsupportedProtocol { .. } => { + event_header = "PeerWithUnsupportedProtocol"; + } NetworkEvent::NewListenAddr(_) => { event_header = "NewListenAddr"; if !cfg!(feature = "local-discovery") { From f86f758711c5c54a8a810b3f18c69f931c1726eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Mar 2024 13:20:27 +0000 Subject: [PATCH 026/205] chore(deps): bump wagoid/commitlint-github-action from 5.4.5 to 6.0.0 Bumps [wagoid/commitlint-github-action](https://github.com/wagoid/commitlint-github-action) from 5.4.5 to 6.0.0. - [Changelog](https://github.com/wagoid/commitlint-github-action/blob/master/CHANGELOG.md) - [Commits](https://github.com/wagoid/commitlint-github-action/compare/5ce82f5d814d4010519d15f0552aec4f17a1e1fe...3c75220e8d20774b68eea48e869a706ff7249b54) --- updated-dependencies: - dependency-name: wagoid/commitlint-github-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 69ee8f93ab..3e6de96b1e 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -44,7 +44,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: wagoid/commitlint-github-action@5ce82f5d814d4010519d15f0552aec4f17a1e1fe + - uses: wagoid/commitlint-github-action@3c75220e8d20774b68eea48e869a706ff7249b54 checks: if: "!startsWith(github.event.head_commit.message, 'chore(release):')" From 523373996931c400d98dca64475fed5c2fbd4852 Mon Sep 17 00:00:00 2001 From: grumbach Date: Mon, 8 Apr 2024 22:01:50 +0900 Subject: [PATCH 027/205] feat: unit testing dag, double spend poisoning tweaks --- .../src/bin/subcommands/wallet/wo_wallet.rs | 2 +- sn_client/src/audit.rs | 257 +---------------- sn_client/src/audit/dag_error.rs | 20 +- sn_client/src/audit/spend_check.rs | 260 ++++++++++++++++++ sn_client/src/audit/spend_dag.rs | 87 +++--- sn_client/src/audit/spend_dag_building.rs | 3 +- sn_client/src/audit/tests/mod.rs | 87 ++++++ sn_client/src/audit/tests/setup.rs | 152 ++++++++++ sn_node/tests/double_spend.rs | 6 +- sn_transfers/benches/reissue.rs | 4 +- .../src/transfers/offline_transfer.rs | 6 +- sn_transfers/src/wallet/hot_wallet.rs | 8 +- 12 files changed, 574 insertions(+), 318 deletions(-) create mode 100644 sn_client/src/audit/spend_check.rs create mode 100644 sn_client/src/audit/tests/mod.rs create mode 100644 sn_client/src/audit/tests/setup.rs diff --git a/sn_cli/src/bin/subcommands/wallet/wo_wallet.rs b/sn_cli/src/bin/subcommands/wallet/wo_wallet.rs index 7a955956d1..73396b9949 100644 --- a/sn_cli/src/bin/subcommands/wallet/wo_wallet.rs +++ b/sn_cli/src/bin/subcommands/wallet/wo_wallet.rs @@ -304,7 +304,7 @@ async fn broadcast_signed_spends( let transfer = OfflineTransfer::from_transaction(signed_spends, tx, change_id, output_details)?; // return the first CashNote (assuming there is only one because we only sent to one recipient) - let cash_note = match &transfer.created_cash_notes[..] { + let cash_note = match &transfer.cash_notes_for_recipient[..] { [cashnote] => cashnote.to_hex()?, [_multiple, ..] => bail!("Multiple CashNotes were returned from the transaction when only one was expected. This is a BUG."), [] =>bail!("No CashNotes were built from the Tx.") diff --git a/sn_client/src/audit.rs b/sn_client/src/audit.rs index 6eac78efa5..aa1482db40 100644 --- a/sn_client/src/audit.rs +++ b/sn_client/src/audit.rs @@ -7,261 +7,12 @@ // permissions and limitations relating to use of the SAFE Network Software. mod dag_error; +mod spend_check; mod spend_dag; mod spend_dag_building; +#[cfg(test)] +mod tests; + pub use dag_error::{DagError, SpendFault}; pub use spend_dag::{SpendDag, SpendDagGet}; - -use super::{ - error::{Error, Result}, - Client, -}; - -use futures::future::join_all; -use sn_networking::{target_arch::Instant, GetRecordError, NetworkError}; -use sn_transfers::{Hash, SignedSpend, SpendAddress, WalletError, WalletResult}; -use std::{collections::BTreeSet, iter::Iterator}; - -impl Client { - /// Verify that a spend is valid on the network. - /// Optionally verify its ancestors as well, all the way to genesis (might take a LONG time) - /// - /// Prints progress on stdout. - /// - /// When verifying all the way back to genesis, it only verifies Spends that are ancestors of the given Spend, - /// ignoring all other branches. - /// - /// This is how the DAG it follows could look like: - /// ```text - /// ... -- - /// \ - /// ... ---- ... -- - /// \ \ - /// Spend0 -> Spend1 -----> Spend2 ---> Spend5 ---> Spend2 ---> Genesis - /// \ / - /// ---> Spend3 ---> Spend6 -> - /// \ / - /// -> Spend4 -> - /// / - /// ... - /// - /// depth0 depth1 depth2 depth3 depth4 depth5 - /// ``` - /// - /// This function will return an error if any spend in the way is invalid. - pub async fn verify_spend_at(&self, addr: SpendAddress, to_genesis: bool) -> WalletResult<()> { - let first_spend = self - .get_spend_from_network(addr) - .await - .map_err(|err| WalletError::CouldNotVerifyTransfer(err.to_string()))?; - - if !to_genesis { - return Ok(()); - } - - // use iteration instead of recursion to avoid stack overflow - let mut txs_to_verify = BTreeSet::from_iter([first_spend.spend.parent_tx]); - let mut depth = 0; - let mut verified_tx = BTreeSet::new(); - let start = Instant::now(); - - while !txs_to_verify.is_empty() { - let mut next_gen_tx = BTreeSet::new(); - - for parent_tx in txs_to_verify { - let parent_tx_hash = parent_tx.hash(); - let parent_keys = parent_tx.inputs.iter().map(|input| input.unique_pubkey); - let addrs_to_verify = parent_keys.map(|k| SpendAddress::from_unique_pubkey(&k)); - debug!("Depth {depth} - Verifying parent Tx : {parent_tx_hash:?} with inputs: {addrs_to_verify:?}"); - - // get all parent spends in parallel - let tasks: Vec<_> = addrs_to_verify - .clone() - .map(|a| self.get_spend_from_network(a)) - .collect(); - let spends = join_all(tasks).await - .into_iter() - .zip(addrs_to_verify.into_iter()) - .map(|(maybe_spend, a)| - maybe_spend.map_err(|err| WalletError::CouldNotVerifyTransfer(format!("at depth {depth} - Failed to get spend {a:?} from network for parent Tx {parent_tx_hash:?}: {err}")))) - .collect::>>()?; - debug!( - "Depth {depth} - Got {:?} spends for parent Tx: {parent_tx_hash:?}", - spends.len() - ); - trace!("Spends for {parent_tx_hash:?} - {spends:?}"); - - // check if we reached the genesis Tx - if parent_tx == sn_transfers::GENESIS_CASHNOTE.src_tx - && spends - .iter() - .all(|s| s.spend.unique_pubkey == sn_transfers::GENESIS_CASHNOTE.id) - && spends.len() == 1 - { - debug!("Depth {depth} - Reached genesis Tx on one branch: {parent_tx_hash:?}"); - verified_tx.insert(parent_tx_hash); - continue; - } - - // verify tx with those spends - parent_tx - .verify_against_inputs_spent(&spends) - .map_err(|err| WalletError::CouldNotVerifyTransfer(format!("at depth {depth} - Failed to verify parent Tx {parent_tx_hash:?}: {err}")))?; - verified_tx.insert(parent_tx_hash); - debug!("Depth {depth} - Verified parent Tx: {parent_tx_hash:?}"); - - // add new parent spends to next gen - next_gen_tx.extend(spends.into_iter().map(|s| s.spend.parent_tx)); - } - - // only verify parents we haven't already verified - txs_to_verify = next_gen_tx - .into_iter() - .filter(|tx| !verified_tx.contains(&tx.hash())) - .collect(); - - depth += 1; - let elapsed = start.elapsed(); - let n = verified_tx.len(); - println!("Now at depth {depth} - Verified {n} transactions in {elapsed:?}"); - } - - let elapsed = start.elapsed(); - let n = verified_tx.len(); - println!("Verified all the way to genesis! Through {depth} generations, verifying {n} transactions in {elapsed:?}"); - Ok(()) - } - - /// This function does the opposite of verify_spend. - /// It recursively follows the descendants of a Spend, all the way to unspent Transaction Outputs (UTXOs). - /// - /// Prints progress on stdout - /// - /// Starting from Genesis, this amounts to Auditing the entire currency. - /// This is how the DAG it follows could look like: - /// - /// ```text - /// -> Spend7 ---> UTXO_11 - /// / - /// Genesis -> Spend1 -----> Spend2 ---> Spend5 ---> UTXO_10 - /// \ - /// ---> Spend3 ---> Spend6 ---> UTXO_9 - /// \ - /// -> Spend4 ---> UTXO_8 - /// - /// gen0 gen1 gen2 gen3 - /// - /// ``` - /// - /// This function will return the UTXOs (Spend addresses not spent yet) - /// Future calls to this function could start from those UTXOs to avoid - /// re-checking all previously checked branches. - pub async fn follow_spend( - &self, - spend_addr: SpendAddress, - ) -> WalletResult> { - let first_spend = self - .get_spend_from_network(spend_addr) - .await - .map_err(|err| WalletError::CouldNotVerifyTransfer(err.to_string()))?; - println!("Generation 0 - Found first spend: {spend_addr:#?}"); - - // use iteration instead of recursion to avoid stack overflow - let mut txs_to_follow = BTreeSet::from_iter([first_spend.spend.spent_tx]); - let mut all_utxos = BTreeSet::new(); - let mut verified_tx = BTreeSet::new(); - let mut gen = 0; - let start = Instant::now(); - - while !txs_to_follow.is_empty() { - let mut next_gen_tx = BTreeSet::new(); - let mut next_gen_spends = BTreeSet::new(); - let mut next_gen_utxos = BTreeSet::new(); - - for descendant_tx in txs_to_follow.iter() { - let descendant_tx_hash = descendant_tx.hash(); - let descendant_keys = descendant_tx - .outputs - .iter() - .map(|output| output.unique_pubkey); - let addrs_to_follow = descendant_keys.map(|k| SpendAddress::from_unique_pubkey(&k)); - debug!("Gen {gen} - Following descendant Tx : {descendant_tx_hash:?} with outputs: {addrs_to_follow:?}"); - - // get all descendant spends in parallel - let tasks: Vec<_> = addrs_to_follow - .clone() - .map(|a| self.get_spend_from_network(a)) - .collect(); - let spends_res = join_all(tasks) - .await - .into_iter() - .zip(addrs_to_follow) - .collect::>(); - - // split spends into utxos and spends - let (utxos, spends) = split_utxos_and_spends(spends_res, gen, descendant_tx_hash)?; - debug!("Gen {gen} - Got {:?} spends and {:?} utxos for descendant Tx: {descendant_tx_hash:?}", spends.len(), utxos.len()); - trace!("Spends for {descendant_tx_hash:?} - {spends:?}"); - next_gen_utxos.extend(utxos); - next_gen_spends.extend( - spends - .iter() - .map(|s| SpendAddress::from_unique_pubkey(&s.spend.unique_pubkey)), - ); - - // add new descendant spends to next gen - next_gen_tx.extend(spends.into_iter().map(|s| s.spend.spent_tx)); - } - - // print stats - gen += 1; - let elapsed = start.elapsed(); - let u = next_gen_utxos.len(); - let s = next_gen_spends.len(); - println!("Generation {gen} - Found {u} UTXOs and {s} Spends in {elapsed:?}"); - debug!("Generation {gen} - UTXOs: {:#?}", next_gen_utxos); - debug!("Generation {gen} - Spends: {:#?}", next_gen_spends); - all_utxos.extend(next_gen_utxos); - - // only verify tx we haven't already verified - verified_tx.extend(txs_to_follow.iter().map(|tx| tx.hash())); - txs_to_follow = next_gen_tx - .into_iter() - .filter(|tx| !verified_tx.contains(&tx.hash())) - .collect(); - } - - let elapsed = start.elapsed(); - let n = all_utxos.len(); - let tx = verified_tx.len(); - println!("Finished auditing! Through {gen} generations, found {n} UTXOs and verified {tx} Transactions in {elapsed:?}"); - Ok(all_utxos) - } -} - -fn split_utxos_and_spends( - spends_res: Vec<(Result, SpendAddress)>, - gen: usize, - descendant_tx_hash: Hash, -) -> WalletResult<(Vec, Vec)> { - let mut utxos = Vec::new(); - let mut spends = Vec::new(); - - for (res, addr) in spends_res { - match res { - Ok(spend) => { - spends.push(spend); - } - Err(Error::Network(NetworkError::GetRecordError(GetRecordError::RecordNotFound))) => { - utxos.push(addr); - } - Err(err) => { - warn!("Error while following spend {addr:?}: {err}"); - return Err(WalletError::CouldNotVerifyTransfer(format!("at gen {gen} - Failed to get spend {addr:?} from network for descendant Tx {descendant_tx_hash:?}: {err}"))); - } - } - } - - Ok((utxos, spends)) -} diff --git a/sn_client/src/audit/dag_error.rs b/sn_client/src/audit/dag_error.rs index d553ed22ee..6fb79953fd 100644 --- a/sn_client/src/audit/dag_error.rs +++ b/sn_client/src/audit/dag_error.rs @@ -28,23 +28,20 @@ pub enum DagError { pub enum SpendFault { #[error("Double Spend at {0:?}")] DoubleSpend(SpendAddress), - #[error("Spend at {addr:?} has missing ancestors at {invalid_ancestor:?}")] + #[error("Spend at {addr:?} has a missing ancestor at {ancestor:?}, until this ancestor is added to the DAG, it cannot be verified")] MissingAncestry { addr: SpendAddress, - invalid_ancestor: SpendAddress, + ancestor: SpendAddress, }, - #[error("Spend at {addr:?} has invalid ancestors at {invalid_ancestor:?}")] - InvalidAncestry { + #[error( + "Spend at {addr:?} has a double spent ancestor at {ancestor:?}, making it unspendable" + )] + DoubleSpentAncestor { addr: SpendAddress, - invalid_ancestor: SpendAddress, + ancestor: SpendAddress, }, #[error("Invalid transaction for spend at {0:?}: {1}")] InvalidTransaction(SpendAddress, String), - #[error("Spend at {addr:?} has an unknown ancestor at {ancestor_addr:?}, until this ancestor is added to the DAG, it cannot be verified")] - UnknownAncestor { - addr: SpendAddress, - ancestor_addr: SpendAddress, - }, #[error("Poisoned ancestry for spend at {0:?}: {1}")] PoisonedAncestry(SpendAddress, String), #[error("Spend at {addr:?} does not descend from given source: {src:?}")] @@ -69,9 +66,8 @@ impl SpendFault { match self { SpendFault::DoubleSpend(addr) | SpendFault::MissingAncestry { addr, .. } - | SpendFault::InvalidAncestry { addr, .. } + | SpendFault::DoubleSpentAncestor { addr, .. } | SpendFault::InvalidTransaction(addr, _) - | SpendFault::UnknownAncestor { addr, .. } | SpendFault::PoisonedAncestry(addr, _) | SpendFault::OrphanSpend { addr, .. } => *addr, } diff --git a/sn_client/src/audit/spend_check.rs b/sn_client/src/audit/spend_check.rs new file mode 100644 index 0000000000..e2df4a4a7b --- /dev/null +++ b/sn_client/src/audit/spend_check.rs @@ -0,0 +1,260 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::{ + error::{Error, Result}, + Client, +}; + +use futures::future::join_all; +use sn_networking::{target_arch::Instant, GetRecordError, NetworkError}; +use sn_transfers::{Hash, SignedSpend, SpendAddress, WalletError, WalletResult}; +use std::{collections::BTreeSet, iter::Iterator}; + +impl Client { + /// Verify that a spend is valid on the network. + /// Optionally verify its ancestors as well, all the way to genesis (might take a LONG time) + /// + /// Prints progress on stdout. + /// + /// When verifying all the way back to genesis, it only verifies Spends that are ancestors of the given Spend, + /// ignoring all other branches. + /// + /// This is how the DAG it follows could look like: + /// ```text + /// ... -- + /// \ + /// ... ---- ... -- + /// \ \ + /// Spend0 -> Spend1 -----> Spend2 ---> Spend5 ---> Spend2 ---> Genesis + /// \ / + /// ---> Spend3 ---> Spend6 -> + /// \ / + /// -> Spend4 -> + /// / + /// ... + /// + /// depth0 depth1 depth2 depth3 depth4 depth5 + /// ``` + /// + /// This function will return an error if any spend in the way is invalid. + pub async fn verify_spend_at(&self, addr: SpendAddress, to_genesis: bool) -> WalletResult<()> { + let first_spend = self + .get_spend_from_network(addr) + .await + .map_err(|err| WalletError::CouldNotVerifyTransfer(err.to_string()))?; + + if !to_genesis { + return Ok(()); + } + + // use iteration instead of recursion to avoid stack overflow + let mut txs_to_verify = BTreeSet::from_iter([first_spend.spend.parent_tx]); + let mut depth = 0; + let mut verified_tx = BTreeSet::new(); + let start = Instant::now(); + + while !txs_to_verify.is_empty() { + let mut next_gen_tx = BTreeSet::new(); + + for parent_tx in txs_to_verify { + let parent_tx_hash = parent_tx.hash(); + let parent_keys = parent_tx.inputs.iter().map(|input| input.unique_pubkey); + let addrs_to_verify = parent_keys.map(|k| SpendAddress::from_unique_pubkey(&k)); + debug!("Depth {depth} - Verifying parent Tx : {parent_tx_hash:?} with inputs: {addrs_to_verify:?}"); + + // get all parent spends in parallel + let tasks: Vec<_> = addrs_to_verify + .clone() + .map(|a| self.get_spend_from_network(a)) + .collect(); + let spends = join_all(tasks).await + .into_iter() + .zip(addrs_to_verify.into_iter()) + .map(|(maybe_spend, a)| + maybe_spend.map_err(|err| WalletError::CouldNotVerifyTransfer(format!("at depth {depth} - Failed to get spend {a:?} from network for parent Tx {parent_tx_hash:?}: {err}")))) + .collect::>>()?; + debug!( + "Depth {depth} - Got {:?} spends for parent Tx: {parent_tx_hash:?}", + spends.len() + ); + trace!("Spends for {parent_tx_hash:?} - {spends:?}"); + + // check if we reached the genesis Tx + if parent_tx == sn_transfers::GENESIS_CASHNOTE.src_tx + && spends + .iter() + .all(|s| s.spend.unique_pubkey == sn_transfers::GENESIS_CASHNOTE.id) + && spends.len() == 1 + { + debug!("Depth {depth} - Reached genesis Tx on one branch: {parent_tx_hash:?}"); + verified_tx.insert(parent_tx_hash); + continue; + } + + // verify tx with those spends + parent_tx + .verify_against_inputs_spent(&spends) + .map_err(|err| WalletError::CouldNotVerifyTransfer(format!("at depth {depth} - Failed to verify parent Tx {parent_tx_hash:?}: {err}")))?; + verified_tx.insert(parent_tx_hash); + debug!("Depth {depth} - Verified parent Tx: {parent_tx_hash:?}"); + + // add new parent spends to next gen + next_gen_tx.extend(spends.into_iter().map(|s| s.spend.parent_tx)); + } + + // only verify parents we haven't already verified + txs_to_verify = next_gen_tx + .into_iter() + .filter(|tx| !verified_tx.contains(&tx.hash())) + .collect(); + + depth += 1; + let elapsed = start.elapsed(); + let n = verified_tx.len(); + println!("Now at depth {depth} - Verified {n} transactions in {elapsed:?}"); + } + + let elapsed = start.elapsed(); + let n = verified_tx.len(); + println!("Verified all the way to genesis! Through {depth} generations, verifying {n} transactions in {elapsed:?}"); + Ok(()) + } + + /// This function does the opposite of verify_spend. + /// It recursively follows the descendants of a Spend, all the way to unspent Transaction Outputs (UTXOs). + /// + /// Prints progress on stdout + /// + /// Starting from Genesis, this amounts to Auditing the entire currency. + /// This is how the DAG it follows could look like: + /// + /// ```text + /// -> Spend7 ---> UTXO_11 + /// / + /// Genesis -> Spend1 -----> Spend2 ---> Spend5 ---> UTXO_10 + /// \ + /// ---> Spend3 ---> Spend6 ---> UTXO_9 + /// \ + /// -> Spend4 ---> UTXO_8 + /// + /// gen0 gen1 gen2 gen3 + /// + /// ``` + /// + /// This function will return the UTXOs (Spend addresses not spent yet) + /// Future calls to this function could start from those UTXOs to avoid + /// re-checking all previously checked branches. + pub async fn follow_spend( + &self, + spend_addr: SpendAddress, + ) -> WalletResult> { + let first_spend = self + .get_spend_from_network(spend_addr) + .await + .map_err(|err| WalletError::CouldNotVerifyTransfer(err.to_string()))?; + println!("Generation 0 - Found first spend: {spend_addr:#?}"); + + // use iteration instead of recursion to avoid stack overflow + let mut txs_to_follow = BTreeSet::from_iter([first_spend.spend.spent_tx]); + let mut all_utxos = BTreeSet::new(); + let mut verified_tx = BTreeSet::new(); + let mut gen = 0; + let start = Instant::now(); + + while !txs_to_follow.is_empty() { + let mut next_gen_tx = BTreeSet::new(); + let mut next_gen_spends = BTreeSet::new(); + let mut next_gen_utxos = BTreeSet::new(); + + for descendant_tx in txs_to_follow.iter() { + let descendant_tx_hash = descendant_tx.hash(); + let descendant_keys = descendant_tx + .outputs + .iter() + .map(|output| output.unique_pubkey); + let addrs_to_follow = descendant_keys.map(|k| SpendAddress::from_unique_pubkey(&k)); + debug!("Gen {gen} - Following descendant Tx : {descendant_tx_hash:?} with outputs: {addrs_to_follow:?}"); + + // get all descendant spends in parallel + let tasks: Vec<_> = addrs_to_follow + .clone() + .map(|a| self.get_spend_from_network(a)) + .collect(); + let spends_res = join_all(tasks) + .await + .into_iter() + .zip(addrs_to_follow) + .collect::>(); + + // split spends into utxos and spends + let (utxos, spends) = split_utxos_and_spends(spends_res, gen, descendant_tx_hash)?; + debug!("Gen {gen} - Got {:?} spends and {:?} utxos for descendant Tx: {descendant_tx_hash:?}", spends.len(), utxos.len()); + trace!("Spends for {descendant_tx_hash:?} - {spends:?}"); + next_gen_utxos.extend(utxos); + next_gen_spends.extend( + spends + .iter() + .map(|s| SpendAddress::from_unique_pubkey(&s.spend.unique_pubkey)), + ); + + // add new descendant spends to next gen + next_gen_tx.extend(spends.into_iter().map(|s| s.spend.spent_tx)); + } + + // print stats + gen += 1; + let elapsed = start.elapsed(); + let u = next_gen_utxos.len(); + let s = next_gen_spends.len(); + println!("Generation {gen} - Found {u} UTXOs and {s} Spends in {elapsed:?}"); + debug!("Generation {gen} - UTXOs: {:#?}", next_gen_utxos); + debug!("Generation {gen} - Spends: {:#?}", next_gen_spends); + all_utxos.extend(next_gen_utxos); + + // only verify tx we haven't already verified + verified_tx.extend(txs_to_follow.iter().map(|tx| tx.hash())); + txs_to_follow = next_gen_tx + .into_iter() + .filter(|tx| !verified_tx.contains(&tx.hash())) + .collect(); + } + + let elapsed = start.elapsed(); + let n = all_utxos.len(); + let tx = verified_tx.len(); + println!("Finished auditing! Through {gen} generations, found {n} UTXOs and verified {tx} Transactions in {elapsed:?}"); + Ok(all_utxos) + } +} + +fn split_utxos_and_spends( + spends_res: Vec<(Result, SpendAddress)>, + gen: usize, + descendant_tx_hash: Hash, +) -> WalletResult<(Vec, Vec)> { + let mut utxos = Vec::new(); + let mut spends = Vec::new(); + + for (res, addr) in spends_res { + match res { + Ok(spend) => { + spends.push(spend); + } + Err(Error::Network(NetworkError::GetRecordError(GetRecordError::RecordNotFound))) => { + utxos.push(addr); + } + Err(err) => { + warn!("Error while following spend {addr:?}: {err}"); + return Err(WalletError::CouldNotVerifyTransfer(format!("at gen {gen} - Failed to get spend {addr:?} from network for descendant Tx {descendant_tx_hash:?}: {err}"))); + } + } + } + + Ok((utxos, spends)) +} diff --git a/sn_client/src/audit/spend_dag.rs b/sn_client/src/audit/spend_dag.rs index 2d63d90b3f..b11d99a424 100644 --- a/sn_client/src/audit/spend_dag.rs +++ b/sn_client/src/audit/spend_dag.rs @@ -126,7 +126,7 @@ impl SpendDag { ); node_idx } - // or upgrade existing utxo to spend + // or upgrade a known but not gathered entry to spend Some(DagEntry::NotGatheredYet(idx)) => { self.spends .insert(spend_addr, DagEntry::Spend(Box::new(spend.clone()), idx)); @@ -324,42 +324,54 @@ impl SpendDag { } /// helper that returns the direct ancestors of a given spend - fn get_ancestor_spends( + /// along with any faults detected + /// On error returns the address of the missing ancestor + fn get_direct_ancestors( &self, spend: &SignedSpend, - ) -> Result, SpendFault> { + ) -> Result<(BTreeSet, BTreeSet), SpendAddress> { let addr = spend.address(); let mut ancestors = BTreeSet::new(); + let mut faults = BTreeSet::new(); for input in spend.spend.parent_tx.inputs.iter() { let ancestor_addr = SpendAddress::from_unique_pubkey(&input.unique_pubkey); match self.spends.get(&ancestor_addr) { Some(DagEntry::Spend(ancestor_spend, _)) => { ancestors.insert(*ancestor_spend.clone()); } - Some(DagEntry::NotGatheredYet(_)) => { - warn!("UnknownAncestor: ancestor {ancestor_addr:?} was not gathered yet for spend {spend:?}"); - return Err(SpendFault::UnknownAncestor { - addr, - ancestor_addr, - }); - } - Some(DagEntry::DoubleSpend(_)) => { - warn!("InvalidAncestry: DoubleSpend ancestor {ancestor_addr:?} for spend {spend:?}"); - return Err(SpendFault::InvalidAncestry { - addr, - invalid_ancestor: ancestor_addr, - }); + Some(DagEntry::NotGatheredYet(_)) | None => { + warn!("Direct ancestor of {spend:?} at {ancestor_addr:?} is missing"); + return Err(ancestor_addr); } - None => { - warn!("MissingAncestry: ancestor {ancestor_addr:?} is unknown for spend {spend:?}"); - return Err(SpendFault::MissingAncestry { + Some(DagEntry::DoubleSpend(multiple_ancestors)) => { + debug!("Direct ancestor for spend {spend:?} at {ancestor_addr:?} is a double spend"); + faults.insert(SpendFault::DoubleSpentAncestor { addr, - invalid_ancestor: ancestor_addr, + ancestor: ancestor_addr, }); + let actual_ancestor: Vec<_> = multiple_ancestors + .iter() + .filter(|(s, _)| s.spend.spent_tx.hash() == spend.spend.parent_tx.hash()) + .map(|(s, _)| s.clone()) + .collect(); + match actual_ancestor.as_slice() { + [ancestor_spend] => { + debug!("Direct ancestor of {spend:?} at {ancestor_addr:?} is a double spend but one of those match our parent_tx hash, using it for verification"); + ancestors.insert(ancestor_spend.clone()); + } + [ancestor1, _ancestor2, ..] => { + warn!("Direct ancestor of {spend:?} at {ancestor_addr:?} is a double spend and mutliple match our parent_tx hash, using the first one for verification"); + ancestors.insert(ancestor1.clone()); + } + [] => { + warn!("Direct ancestor of {spend:?} at {ancestor_addr:?} is a double spend and none of them match the spend parent_tx, which means the parent for this spend is missing!"); + return Err(ancestor_addr); + } + } } } } - Ok(ancestors) + Ok((ancestors, faults)) } /// helper that returns all the descendants (recursively all the way to UTXOs) of a given spend @@ -543,35 +555,34 @@ impl SpendDag { } // get the ancestors of this spend - let ancestor_spends = match self.get_ancestor_spends(spend) { + let (ancestor_spends, faults) = match self.get_direct_ancestors(spend) { Ok(a) => a, - Err(fault) => { - debug!("Failed to get ancestor spends of {addr:?}: {fault}"); - recorded_faults.insert(fault.clone()); - - // if ancestry is invalid, poison all the descendants - let poison = format!("ancestry issue: {fault}"); + Err(missing_ancestor) => { + debug!("Failed to get ancestor spends of {addr:?} as ancestor at {missing_ancestor:?} is missing"); + recorded_faults.insert(SpendFault::MissingAncestry { + addr, + ancestor: missing_ancestor, + }); + + let poison = format!("missing ancestor at: {missing_ancestor:?}"); let descendants_faults = self.poison_all_descendants(spend, poison)?; recorded_faults.extend(descendants_faults); return Ok(recorded_faults); } }; + recorded_faults.extend(faults); // verify the tx - match spend + if let Err(e) = spend .spend .parent_tx .verify_against_inputs_spent(&ancestor_spends) { - Ok(_) => (), - Err(e) => { - recorded_faults.insert(SpendFault::InvalidTransaction(addr, format!("{e}"))); - - // mark this spend's descendants as poisoned if tx is invalid - let poison = format!("ancestor transaction was poisoned at: {addr:?}: {e}"); - let descendants_faults = self.poison_all_descendants(spend, poison)?; - recorded_faults.extend(descendants_faults); - } + warn!("Parent Tx verfication failed for spend at: {addr:?}: {e}"); + recorded_faults.insert(SpendFault::InvalidTransaction(addr, format!("{e}"))); + let poison = format!("ancestor transaction was poisoned at: {addr:?}: {e}"); + let descendants_faults = self.poison_all_descendants(spend, poison)?; + recorded_faults.extend(descendants_faults); } Ok(recorded_faults) diff --git a/sn_client/src/audit/spend_dag_building.rs b/sn_client/src/audit/spend_dag_building.rs index 0a269b62c2..6fb7737523 100644 --- a/sn_client/src/audit/spend_dag_building.rs +++ b/sn_client/src/audit/spend_dag_building.rs @@ -6,8 +6,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use super::{Client, SpendDag}; -use crate::Error; +use crate::{Client, Error, SpendDag}; use futures::{future::join_all, StreamExt}; use sn_networking::{GetRecordError, NetworkError}; diff --git a/sn_client/src/audit/tests/mod.rs b/sn_client/src/audit/tests/mod.rs new file mode 100644 index 0000000000..5a9d28149e --- /dev/null +++ b/sn_client/src/audit/tests/mod.rs @@ -0,0 +1,87 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +mod setup; + +use std::collections::BTreeSet; + +use setup::MockNetwork; + +use eyre::Result; + +use crate::{SpendDag, SpendFault}; + +#[test] +fn test_spend_dag_verify_valid_simple() -> Result<()> { + let mut net = MockNetwork::genesis()?; + let genesis = net.genesis_spend; + + let owner1 = net.new_pk_with_balance(100)?; + let owner2 = net.new_pk_with_balance(0)?; + let owner3 = net.new_pk_with_balance(0)?; + let owner4 = net.new_pk_with_balance(0)?; + let owner5 = net.new_pk_with_balance(0)?; + let owner6 = net.new_pk_with_balance(0)?; + + net.send(&owner1, &owner2, 100)?; + net.send(&owner2, &owner3, 100)?; + net.send(&owner3, &owner4, 100)?; + net.send(&owner4, &owner5, 100)?; + net.send(&owner5, &owner6, 100)?; + + let mut dag = SpendDag::new(genesis); + for spend in net.spends { + dag.insert(spend.address(), spend.clone()); + } + + assert_eq!(dag.verify(&genesis), Ok(BTreeSet::new())); + Ok(()) +} + +#[test] +fn test_spend_dag_double_spend_detection() -> Result<()> { + let mut net = MockNetwork::genesis()?; + let genesis = net.genesis_spend; + + let owner1 = net.new_pk_with_balance(100)?; + let owner2a = net.new_pk_with_balance(0)?; + let owner2b = net.new_pk_with_balance(0)?; + + let cn_to_reuse = net + .wallets + .get(&owner1) + .expect("owner1 wallet to exist") + .cn + .clone(); + let spend1_addr = net.send(&owner1, &owner2a, 100)?; + net.wallets + .get_mut(&owner1) + .expect("owner1 wallet to still exist") + .cn = cn_to_reuse; + let spend2_addr = net.send(&owner1, &owner2b, 100)?; + + let mut dag = SpendDag::new(genesis); + for spend in net.spends { + dag.insert(spend.address(), spend.clone()); + } + dag.record_faults(&genesis)?; + + assert_eq!( + spend1_addr, spend2_addr, + "both spends should be at the same address" + ); + assert_eq!(spend1_addr.len(), 1, "there should only be one spend"); + let double_spent = spend1_addr.first().expect("spend1_addr to have an element"); + let expected = BTreeSet::from_iter([SpendFault::DoubleSpend(*double_spent)]); + assert_eq!( + dag.get_spend_faults(double_spent), + expected, + "DAG should have detected double spend" + ); + Ok(()) +} diff --git a/sn_client/src/audit/tests/setup.rs b/sn_client/src/audit/tests/setup.rs new file mode 100644 index 0000000000..cdc7caff80 --- /dev/null +++ b/sn_client/src/audit/tests/setup.rs @@ -0,0 +1,152 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use std::collections::{BTreeMap, BTreeSet}; + +use bls::SecretKey; +use eyre::{eyre, Result}; +use sn_transfers::{ + CashNote, DerivationIndex, Hash, MainPubkey, MainSecretKey, NanoTokens, OfflineTransfer, + SignedSpend, SpendAddress, GENESIS_CASHNOTE, GENESIS_CASHNOTE_SK, +}; +use xor_name::XorName; + +pub struct MockWallet { + pub sk: MainSecretKey, + pub cn: Vec, +} + +pub struct MockNetwork { + pub genesis_spend: SpendAddress, + pub spends: BTreeSet, + pub wallets: BTreeMap, +} + +impl MockNetwork { + pub fn genesis() -> Result { + let mut rng = rand::thread_rng(); + let placeholder = SpendAddress::new(XorName::random(&mut rng)); + let mut net = MockNetwork { + genesis_spend: placeholder, + spends: BTreeSet::new(), + wallets: BTreeMap::new(), + }; + + // create genesis wallet + let genesis_cn = GENESIS_CASHNOTE.clone(); + let genesis_sk = MainSecretKey::new( + SecretKey::from_hex(GENESIS_CASHNOTE_SK) + .map_err(|e| eyre!("failed to parse genesis pk: {e}"))?, + ); + let genesis_pk = genesis_sk.main_pubkey(); + net.wallets.insert( + genesis_pk, + MockWallet { + sk: genesis_sk, + cn: vec![genesis_cn], + }, + ); + + // spend genesis + let everything = GENESIS_CASHNOTE + .value() + .map_err(|e| eyre!("invalid genesis cashnote: {e}"))? + .as_nano(); + let spent_addrs = net + .send(&genesis_pk, &genesis_pk, everything) + .map_err(|e| eyre!("failed to send genesis: {e}"))?; + net.genesis_spend = match spent_addrs.as_slice() { + [one] => *one, + _ => { + return Err(eyre!( + "Expected Genesis spend to be unique but got {spent_addrs:?}" + )) + } + }; + + Ok(net) + } + + pub fn new_pk_with_balance(&mut self, balance: u64) -> Result { + let owner = MainSecretKey::new(SecretKey::random()); + let owner_pk = owner.main_pubkey(); + self.wallets.insert( + owner_pk, + MockWallet { + sk: owner, + cn: Vec::new(), + }, + ); + + if balance > 0 { + let genesis_pk = GENESIS_CASHNOTE.main_pubkey(); + self.send(genesis_pk, &owner_pk, balance) + .map_err(|e| eyre!("failed to get money from genesis: {e}"))?; + } + Ok(owner_pk) + } + + pub fn send( + &mut self, + from: &MainPubkey, + to: &MainPubkey, + amount: u64, + ) -> Result> { + let mut rng = rand::thread_rng(); + let from_wallet = self + .wallets + .get(from) + .ok_or_else(|| eyre!("from wallet not found: {from:?}"))?; + let to_wallet = self + .wallets + .get(to) + .ok_or_else(|| eyre!("to wallet not found: {to:?}"))?; + + // perform offline transfer + let cash_notes_with_keys = from_wallet + .cn + .clone() + .into_iter() + .map(|cn| Ok((cn.clone(), Some(cn.derived_key(&from_wallet.sk)?)))) + .collect::>() + .map_err(|e| eyre!("could not get cashnotes for transfer: {e}"))?; + let recipient = vec![( + NanoTokens::from(amount), + to_wallet.sk.main_pubkey(), + DerivationIndex::random(&mut rng), + )]; + let transfer = OfflineTransfer::new( + cash_notes_with_keys, + recipient, + from_wallet.sk.main_pubkey(), + Hash::default(), + ) + .map_err(|e| eyre!("failed to create transfer: {}", e))?; + let spends = transfer.all_spend_requests; + + // update wallets + let mut updated_from_wallet_cns = from_wallet.cn.clone(); + updated_from_wallet_cns.retain(|cn| { + !spends + .iter() + .any(|s| s.unique_pubkey() == &cn.unique_pubkey()) + }); + updated_from_wallet_cns.extend(transfer.change_cash_note); + self.wallets + .entry(*from) + .and_modify(|w| w.cn = updated_from_wallet_cns); + self.wallets + .entry(*to) + .and_modify(|w| w.cn.extend(transfer.cash_notes_for_recipient)); + + // update network spends + let spent_addrs = spends.iter().map(|s| s.address()).collect(); + self.spends.extend(spends); + Ok(spent_addrs) + } +} diff --git a/sn_node/tests/double_spend.rs b/sn_node/tests/double_spend.rs index d861e2b5d8..990eb0cff7 100644 --- a/sn_node/tests/double_spend.rs +++ b/sn_node/tests/double_spend.rs @@ -73,8 +73,8 @@ async fn cash_note_transfer_double_spend_fail() -> Result<()> { // check the CashNotes, it should fail info!("Verifying the transfers from first wallet..."); - let cash_notes_for_2: Vec<_> = transfer_to_2.created_cash_notes.clone(); - let cash_notes_for_3: Vec<_> = transfer_to_3.created_cash_notes.clone(); + let cash_notes_for_2: Vec<_> = transfer_to_2.cash_notes_for_recipient.clone(); + let cash_notes_for_3: Vec<_> = transfer_to_3.cash_notes_for_recipient.clone(); let could_err1 = client.verify_cashnote(&cash_notes_for_2[0]).await; let could_err2 = client.verify_cashnote(&cash_notes_for_3[0]).await; @@ -125,7 +125,7 @@ async fn genesis_double_spend_fail() -> Result<()> { assert!(res.is_ok()); // put the bad cashnote in the first wallet - first_wallet.deposit_and_store_to_disk(&transfer.created_cash_notes)?; + first_wallet.deposit_and_store_to_disk(&transfer.cash_notes_for_recipient)?; // now try to spend this illegitimate cashnote (direct descendant of double spent genesis) let (genesis_cashnote_and_others, exclusive_access) = first_wallet.available_cash_notes()?; diff --git a/sn_transfers/benches/reissue.rs b/sn_transfers/benches/reissue.rs index ead6cde1e1..b2c8cfc900 100644 --- a/sn_transfers/benches/reissue.rs +++ b/sn_transfers/benches/reissue.rs @@ -115,12 +115,12 @@ fn bench_reissue_100_to_1(c: &mut Criterion) { // prepare to send all of those cashnotes back to our starting_main_key let total_amount = offline_transfer - .created_cash_notes + .cash_notes_for_recipient .iter() .map(|cn| cn.value().unwrap().as_nano()) .sum(); let many_cashnotes = offline_transfer - .created_cash_notes + .cash_notes_for_recipient .into_iter() .map(|cn| { let derivation_index = cn.derivation_index(); diff --git a/sn_transfers/src/transfers/offline_transfer.rs b/sn_transfers/src/transfers/offline_transfer.rs index d1f478ffa4..ca44440740 100644 --- a/sn_transfers/src/transfers/offline_transfer.rs +++ b/sn_transfers/src/transfers/offline_transfer.rs @@ -31,7 +31,7 @@ pub struct OfflineTransfer { /// The cash_notes that were created containing /// the tokens sent to respective recipient. #[debug(skip)] - pub created_cash_notes: Vec, + pub cash_notes_for_recipient: Vec, /// The cash_note holding surplus tokens after /// spending the necessary input cash_notes. #[debug(skip)] @@ -70,7 +70,7 @@ impl OfflineTransfer { Ok(Self { tx, - created_cash_notes, + cash_notes_for_recipient: created_cash_notes, change_cash_note, all_spend_requests: signed_spends.into_iter().collect(), }) @@ -345,7 +345,7 @@ fn create_offline_transfer_with( Ok(OfflineTransfer { tx, - created_cash_notes, + cash_notes_for_recipient: created_cash_notes, change_cash_note, all_spend_requests, }) diff --git a/sn_transfers/src/wallet/hot_wallet.rs b/sn_transfers/src/wallet/hot_wallet.rs index bcf23d97ae..7d738952d1 100644 --- a/sn_transfers/src/wallet/hot_wallet.rs +++ b/sn_transfers/src/wallet/hot_wallet.rs @@ -335,7 +335,7 @@ impl HotWallet { reason_hash, )?; - let created_cash_notes = transfer.created_cash_notes.clone(); + let created_cash_notes = transfer.cash_notes_for_recipient.clone(); self.update_local_wallet(transfer, exclusive_access)?; @@ -354,7 +354,7 @@ impl HotWallet { let transfer = OfflineTransfer::from_transaction(signed_spends, tx, change_id, output_details)?; - let created_cash_notes = transfer.created_cash_notes.clone(); + let created_cash_notes = transfer.cash_notes_for_recipient.clone(); trace!("Trying to lock wallet to get available cash_notes..."); // lock and load from disk to make sure we're up to date and others can't modify the wallet concurrently @@ -436,14 +436,14 @@ impl HotWallet { )?; trace!( "local_send_storage_payment created offline_transfer with {} cashnotes in {:?}", - offline_transfer.created_cash_notes.len(), + offline_transfer.cash_notes_for_recipient.len(), start.elapsed() ); let start = Instant::now(); // cache transfer payments in the wallet let mut cashnotes_to_use: HashSet = offline_transfer - .created_cash_notes + .cash_notes_for_recipient .iter() .cloned() .collect(); From 798f28effaaff71a6e9d6198ac1154967bba1744 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 9 Apr 2024 15:53:51 +0530 Subject: [PATCH 028/205] feat(cli): return the files upload summary after a successful files upload --- sn_cli/src/files/chunk_manager.rs | 16 +++++++--- sn_cli/src/files/files_uploader.rs | 51 +++++++++++++++++++++--------- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/sn_cli/src/files/chunk_manager.rs b/sn_cli/src/files/chunk_manager.rs index d2b0415f7a..6dc81f0748 100644 --- a/sn_cli/src/files/chunk_manager.rs +++ b/sn_cli/src/files/chunk_manager.rs @@ -69,7 +69,7 @@ pub struct ChunkManager { artifacts_dir: PathBuf, files_to_chunk: Vec<(OsString, PathXorName, PathBuf)>, chunks: BTreeMap, - completed_files: Vec<(OsString, ChunkAddress)>, + completed_files: Vec<(PathBuf, OsString, ChunkAddress)>, resumed_chunk_count: usize, resumed_files_count: usize, } @@ -185,6 +185,7 @@ impl ChunkManager { let completed_files = self.chunks.iter().filter_map(|(_, chunked_file)| { if chunked_file.chunks.is_empty() { Some(( + chunked_file.file_path.clone(), chunked_file.file_name.clone(), chunked_file.head_chunk_address, )) @@ -391,6 +392,7 @@ impl ChunkManager { trace!("removed {path_xor:?} from chunks list"); self.completed_files.push(( + chunked_file.file_path.clone(), chunked_file.file_name.clone(), chunked_file.head_chunk_address, )); @@ -430,15 +432,21 @@ impl ChunkManager { /// Return the filename and the file's Xor address if all their chunks has been marked as /// completed - pub(crate) fn completed_files(&self) -> &Vec<(OsString, ChunkAddress)> { + pub(crate) fn completed_files(&self) -> &Vec<(PathBuf, OsString, ChunkAddress)> { &self.completed_files } /// Return the list of Filenames that have some chunks that are yet to be marked as completed. - pub(crate) fn incomplete_files(&self) -> Vec<&OsString> { + pub(crate) fn incomplete_files(&self) -> Vec<(&PathBuf, &OsString, &ChunkAddress)> { self.chunks .values() - .map(|chunked_file| &chunked_file.file_name) + .map(|chunked_file| { + ( + &chunked_file.file_path, + &chunked_file.file_name, + &chunked_file.head_chunk_address, + ) + }) .collect() } diff --git a/sn_cli/src/files/files_uploader.rs b/sn_cli/src/files/files_uploader.rs index 3242f8f711..5e5bc4c09d 100644 --- a/sn_cli/src/files/files_uploader.rs +++ b/sn_cli/src/files/files_uploader.rs @@ -6,6 +6,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. +use super::get_progress_bar; use crate::ChunkManager; use bytes::Bytes; use color_eyre::{eyre::eyre, Report, Result}; @@ -16,8 +17,9 @@ use sn_client::{ transfers::{TransferError, WalletError}, Client, Error as ClientError, UploadCfg, UploadEvent, UploadSummary, Uploader, }; -use sn_protocol::storage::Chunk; +use sn_protocol::storage::{Chunk, ChunkAddress}; use std::{ + ffi::OsString, path::{Path, PathBuf}, time::{Duration, Instant}, }; @@ -26,7 +28,15 @@ use tracing::{debug, error, info, warn}; use walkdir::{DirEntry, WalkDir}; use xor_name::XorName; -use super::get_progress_bar; +/// The result of a successful files upload. +pub struct FilesUploadSummary { + /// The cost and count summary of the upload. + pub upload_summary: UploadSummary, + /// The list of completed files (FilePath, FileName, HeadChunkAddress) + pub completed_files: Vec<(PathBuf, OsString, ChunkAddress)>, + /// The list of incomplete files (FilePath, FileName, HeadChunkAddress) + pub incomplete_files: Vec<(PathBuf, OsString, ChunkAddress)>, +} /// Combines the `Uploader` along with the `ChunkManager` pub struct FilesUploader { @@ -79,7 +89,7 @@ impl FilesUploader { self } - pub async fn start_upload(self) -> Result { + pub async fn start_upload(self) -> Result { let mut chunk_manager = ChunkManager::new(&self.root_dir); let chunks_to_upload = self.get_chunks_to_upload(&mut chunk_manager).await?; let chunks_to_upload_len = chunks_to_upload.len(); @@ -119,7 +129,7 @@ impl FilesUploader { ); } - let summary = match uploader.start_upload().await { + let upload_sum = match uploader.start_upload().await { Ok(summary) => summary, Err(ClientError::Wallet(WalletError::Transfer(TransferError::NotEnoughBalance( available, @@ -132,32 +142,43 @@ impl FilesUploader { } Err(err) => return Err(eyre!("Failed to upload chunk batch: {err}")), }; - events_handle.await??; + let chunk_manager = events_handle.await??; let elapsed = Self::msg_format_elapsed_time(now.elapsed()); println!( "Among {chunks_to_upload_len} chunks, found {} already existed in network, uploaded \ the leftover {} chunks in {elapsed}", - summary.skipped_count, summary.uploaded_count, + upload_sum.skipped_count, upload_sum.uploaded_count, ); info!( "Among {chunks_to_upload_len} chunks, found {} already existed in network, uploaded \ the leftover {} chunks in {elapsed}", - summary.skipped_count, summary.uploaded_count, + upload_sum.skipped_count, upload_sum.uploaded_count, ); println!("**************************************"); println!("* Payment Details *"); println!("**************************************"); println!( "Made payment of {:?} for {} chunks", - summary.storage_cost, summary.uploaded_count + upload_sum.storage_cost, upload_sum.uploaded_count ); println!( "Made payment of {:?} for royalties fees", - summary.royalty_fees + upload_sum.royalty_fees ); - println!("New wallet balance: {}", summary.final_balance); - + println!("New wallet balance: {}", upload_sum.final_balance); + + let summary = FilesUploadSummary { + upload_summary: upload_sum, + completed_files: chunk_manager.completed_files().clone(), + incomplete_files: chunk_manager + .incomplete_files() + .into_iter() + .map(|(path, file_name, head_address)| { + (path.clone(), file_name.clone(), *head_address) + }) + .collect(), + }; Ok(summary) } @@ -246,7 +267,7 @@ impl FilesUploader { make_data_public: bool, chunks_to_upload_len: usize, mut upload_event_rx: Receiver, - ) -> Result>> { + ) -> Result>> { let progress_bar = get_progress_bar(chunks_to_upload_len as u64)?; let handle = tokio::spawn(async move { let mut upload_terminated_with_error = false; @@ -279,7 +300,7 @@ impl FilesUploader { if upload_terminated_with_error { error!("Got UploadEvent::Error inside upload event loop"); } else { - for file_name in chunk_manager.incomplete_files() { + for (_, file_name, _) in chunk_manager.incomplete_files() { if let Some(file_name) = file_name.to_str() { println!("Unverified file \"{file_name}\", suggest to re-upload again."); info!("Unverified {file_name}"); @@ -294,7 +315,7 @@ impl FilesUploader { Self::print_completed_file_list(&chunk_manager); } - Ok::<_, Report>(()) + Ok::<_, Report>(chunk_manager) }); Ok(handle) @@ -323,7 +344,7 @@ impl FilesUploader { } fn print_completed_file_list(chunk_manager: &ChunkManager) { - for (file_name, addr) in chunk_manager.completed_files() { + for (_, file_name, addr) in chunk_manager.completed_files() { let hex_addr = addr.to_hex(); if let Some(file_name) = file_name.to_str() { println!("\"{file_name}\" {hex_addr}"); From 1d4761a731f2c8b440c69284c81a5e9e94aae77d Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 9 Apr 2024 17:31:00 +0530 Subject: [PATCH 029/205] feat(cli): implement FilesUploadStatusNotifier trait for lib code This would allow users of this struct to override the default stdout printing and instead provide thier own custom impl --- sn_cli/src/files.rs | 2 +- sn_cli/src/files/files_uploader.rs | 321 ++++++++++++++++++++--------- sn_cli/src/lib.rs | 4 +- 3 files changed, 227 insertions(+), 100 deletions(-) diff --git a/sn_cli/src/files.rs b/sn_cli/src/files.rs index e9b3a1346b..79b12cfc22 100644 --- a/sn_cli/src/files.rs +++ b/sn_cli/src/files.rs @@ -15,7 +15,7 @@ mod upload; pub use chunk_manager::ChunkManager; pub use download::{download_file, download_files}; pub use estimate::Estimator; -pub use files_uploader::FilesUploader; +pub use files_uploader::{FilesUploadStatusNotifier, FilesUploader}; pub use upload::{UploadedFile, UPLOADED_FILES}; use color_eyre::Result; diff --git a/sn_cli/src/files/files_uploader.rs b/sn_cli/src/files/files_uploader.rs index 5e5bc4c09d..59c332947c 100644 --- a/sn_cli/src/files/files_uploader.rs +++ b/sn_cli/src/files/files_uploader.rs @@ -38,30 +38,60 @@ pub struct FilesUploadSummary { pub incomplete_files: Vec<(PathBuf, OsString, ChunkAddress)>, } +/// A trait designed to customize the standard output behavior for file upload processes. +pub trait FilesUploadStatusNotifier: Send { + fn collect_entries(&mut self, entries_iter: Vec); + fn collect_paths(&mut self, path: &Path); + fn on_verifying_uploaded_chunks_init(&self, chunks_len: usize); + fn on_verifying_uploaded_chunks_success( + &self, + completed_files: &[(PathBuf, OsString, ChunkAddress)], + make_data_public: bool, + ); + fn on_verifying_uploaded_chunks_failure(&self, failed_chunks_len: usize); + fn on_failed_to_upload_all_files( + &self, + incomplete_files: Vec<(&PathBuf, &OsString, &ChunkAddress)>, + completed_files: &[(PathBuf, OsString, ChunkAddress)], + make_data_public: bool, + ); + fn on_chunking_complete( + &self, + upload_cfg: &UploadCfg, + make_data_public: bool, + chunks_to_upload_len: usize, + ); + fn on_upload_complete( + &self, + upload_sum: &UploadSummary, + elapsed_time: Duration, + chunks_to_upload_len: usize, + ); +} + /// Combines the `Uploader` along with the `ChunkManager` pub struct FilesUploader { client: Client, root_dir: PathBuf, - - // entries to upload + /// entries to upload entries_to_upload: Vec, - // todo: remove this in favour of an trait that abstracts the std out printing. - file_paths_to_print: Vec, - - // config + /// The status notifier that can be overridden to perform custom actions instead of printing things to stdout. + status_notifier: Option>, + /// config make_data_public: bool, upload_cfg: UploadCfg, } impl FilesUploader { pub fn new(client: Client, root_dir: PathBuf) -> Self { + let status_notifier = Box::new(StdOutPrinter { + file_paths_to_print: Default::default(), + }); Self { client, root_dir, - entries_to_upload: Default::default(), - file_paths_to_print: Default::default(), - + status_notifier: Some(status_notifier), make_data_public: false, upload_cfg: Default::default(), } @@ -77,25 +107,44 @@ impl FilesUploader { self } + /// Override the default status notifier. By default we print things to stdout. + pub fn set_status_notifier( + mut self, + status_notifier: Box, + ) -> Self { + self.status_notifier = Some(status_notifier); + self + } + pub fn insert_entries(mut self, entries_iter: impl IntoIterator) -> Self { self.entries_to_upload.extend(entries_iter); self } pub fn insert_path(mut self, path: &Path) -> Self { - self.file_paths_to_print.push(path.to_path_buf()); + if let Some(notifier) = &mut self.status_notifier { + notifier.collect_paths(path); + } let entries = WalkDir::new(path).into_iter().flatten(); self.entries_to_upload.extend(entries); self } - pub async fn start_upload(self) -> Result { + pub async fn start_upload(mut self) -> Result { let mut chunk_manager = ChunkManager::new(&self.root_dir); let chunks_to_upload = self.get_chunks_to_upload(&mut chunk_manager).await?; let chunks_to_upload_len = chunks_to_upload.len(); - let now = Instant::now(); + // Notify on chunking complete + if let Some(notifier) = &self.status_notifier { + notifier.on_chunking_complete( + &self.upload_cfg, + self.make_data_public, + chunks_to_upload_len, + ); + } + let now = Instant::now(); let mut uploader = Uploader::new(self.client, self.root_dir); uploader.set_upload_cfg(self.upload_cfg); uploader.insert_chunk_paths(chunks_to_upload); @@ -105,30 +154,9 @@ impl FilesUploader { self.make_data_public, chunks_to_upload_len, uploader.get_event_receiver(), + self.status_notifier.take(), )?; - for path in self.file_paths_to_print.iter() { - debug!( - "Uploading file(s) from {path:?} batch size {:?} will verify?: {}", - self.upload_cfg.batch_size, self.upload_cfg.verify_store - ); - if self.make_data_public { - info!("{path:?} will be made public and linkable"); - println!("{path:?} will be made public and linkable"); - } - } - if self.file_paths_to_print.len() == 1 { - println!( - "Splitting and uploading {:?} into {chunks_to_upload_len} chunks", - self.file_paths_to_print[0] - ); - } else { - println!( - "Splitting and uploading {:?} into {chunks_to_upload_len} chunks", - self.file_paths_to_print - ); - } - let upload_sum = match uploader.start_upload().await { Ok(summary) => summary, Err(ClientError::Wallet(WalletError::Transfer(TransferError::NotEnoughBalance( @@ -142,31 +170,13 @@ impl FilesUploader { } Err(err) => return Err(eyre!("Failed to upload chunk batch: {err}")), }; - let chunk_manager = events_handle.await??; + let (chunk_manager, status_notifier) = events_handle.await??; + self.status_notifier = status_notifier; - let elapsed = Self::msg_format_elapsed_time(now.elapsed()); - println!( - "Among {chunks_to_upload_len} chunks, found {} already existed in network, uploaded \ - the leftover {} chunks in {elapsed}", - upload_sum.skipped_count, upload_sum.uploaded_count, - ); - info!( - "Among {chunks_to_upload_len} chunks, found {} already existed in network, uploaded \ - the leftover {} chunks in {elapsed}", - upload_sum.skipped_count, upload_sum.uploaded_count, - ); - println!("**************************************"); - println!("* Payment Details *"); - println!("**************************************"); - println!( - "Made payment of {:?} for {} chunks", - upload_sum.storage_cost, upload_sum.uploaded_count - ); - println!( - "Made payment of {:?} for royalties fees", - upload_sum.royalty_fees - ); - println!("New wallet balance: {}", upload_sum.final_balance); + // Notify on upload complete + if let Some(notifier) = &self.status_notifier { + notifier.on_upload_complete(&upload_sum, now.elapsed(), chunks_to_upload_len); + } let summary = FilesUploadSummary { upload_summary: upload_sum, @@ -204,10 +214,11 @@ impl FilesUploader { self.make_data_public, )?; - println!( - "Files upload attempted previously, verifying {} chunks", - chunks.len() - ); + // Notify on verification init + if let Some(notifier) = &self.status_notifier { + notifier.on_verifying_uploaded_chunks_init(chunks.len()); + } + let failed_chunks = self.verify_uploaded_chunks(&chunks).await?; chunk_manager.mark_completed( @@ -218,17 +229,20 @@ impl FilesUploader { )?; if failed_chunks.is_empty() { - println!("All files were already uploaded and verified"); - Self::print_uploaded_msg(self.make_data_public); - - if chunk_manager.completed_files().is_empty() { - println!("chunk_manager doesn't have any verified_files, nor any failed_chunks to re-upload."); + // Notify on verification success + if let Some(notifier) = &self.status_notifier { + notifier.on_verifying_uploaded_chunks_success( + chunk_manager.completed_files(), + self.make_data_public, + ); } - Self::print_completed_file_list(chunk_manager); return Ok(vec![]); } - println!("{} chunks were uploaded in the past but failed to verify. Will attempt to upload them again...", failed_chunks.len()); + // Notify on verification failure + if let Some(notifier) = &self.status_notifier { + notifier.on_verifying_uploaded_chunks_failure(failed_chunks.len()); + } failed_chunks }; // shuffle the chunks @@ -262,12 +276,15 @@ impl FilesUploader { Ok(failed_chunks) } + #[allow(clippy::type_complexity)] fn spawn_upload_events_handler( mut chunk_manager: ChunkManager, make_data_public: bool, chunks_to_upload_len: usize, mut upload_event_rx: Receiver, - ) -> Result>> { + status_notifier: Option>, + ) -> Result>)>>> + { let progress_bar = get_progress_bar(chunks_to_upload_len as u64)?; let handle = tokio::spawn(async move { let mut upload_terminated_with_error = false; @@ -300,51 +317,149 @@ impl FilesUploader { if upload_terminated_with_error { error!("Got UploadEvent::Error inside upload event loop"); } else { - for (_, file_name, _) in chunk_manager.incomplete_files() { - if let Some(file_name) = file_name.to_str() { - println!("Unverified file \"{file_name}\", suggest to re-upload again."); - info!("Unverified {file_name}"); - } else { - println!("Unverified file \"{file_name:?}\", suggest to re-upload again."); - info!("Unverified file {file_name:?}"); - } + // Notify on upload failure + if let Some(notifier) = &status_notifier { + notifier.on_failed_to_upload_all_files( + chunk_manager.incomplete_files(), + chunk_manager.completed_files(), + make_data_public, + ); } - - // log uploaded file information - Self::print_uploaded_msg(make_data_public); - Self::print_completed_file_list(&chunk_manager); } - Ok::<_, Report>(chunk_manager) + Ok::<_, Report>((chunk_manager, status_notifier)) }); Ok(handle) } +} + +/// The default +struct StdOutPrinter { + file_paths_to_print: Vec, +} - fn msg_format_elapsed_time(elapsed_time: Duration) -> String { +impl FilesUploadStatusNotifier for StdOutPrinter { + fn collect_entries(&mut self, _entries_iter: Vec) {} + + fn collect_paths(&mut self, path: &Path) { + self.file_paths_to_print.push(path.to_path_buf()); + } + + fn on_verifying_uploaded_chunks_init(&self, chunks_len: usize) { + println!("Files upload attempted previously, verifying {chunks_len} chunks",); + } + + fn on_verifying_uploaded_chunks_success( + &self, + completed_files: &[(PathBuf, OsString, ChunkAddress)], + make_data_public: bool, + ) { + println!("All files were already uploaded and verified"); + Self::print_uploaded_msg(make_data_public); + + if completed_files.is_empty() { + println!("chunk_manager doesn't have any verified_files, nor any failed_chunks to re-upload."); + } + Self::print_completed_file_list(completed_files); + } + + fn on_verifying_uploaded_chunks_failure(&self, failed_chunks_len: usize) { + println!("{failed_chunks_len} chunks were uploaded in the past but failed to verify. Will attempt to upload them again..."); + } + + fn on_failed_to_upload_all_files( + &self, + incomplete_files: Vec<(&PathBuf, &OsString, &ChunkAddress)>, + completed_files: &[(PathBuf, OsString, ChunkAddress)], + make_data_public: bool, + ) { + for (_, file_name, _) in incomplete_files { + if let Some(file_name) = file_name.to_str() { + println!("Unverified file \"{file_name}\", suggest to re-upload again."); + info!("Unverified {file_name}"); + } else { + println!("Unverified file \"{file_name:?}\", suggest to re-upload again."); + info!("Unverified file {file_name:?}"); + } + } + + // log uploaded file information + Self::print_uploaded_msg(make_data_public); + Self::print_completed_file_list(completed_files); + } + + fn on_chunking_complete( + &self, + upload_cfg: &UploadCfg, + make_data_public: bool, + chunks_to_upload_len: usize, + ) { + for path in self.file_paths_to_print.iter() { + debug!( + "Uploading file(s) from {path:?} batch size {:?} will verify?: {}", + upload_cfg.batch_size, upload_cfg.verify_store + ); + if make_data_public { + info!("{path:?} will be made public and linkable"); + println!("{path:?} will be made public and linkable"); + } + } + if self.file_paths_to_print.len() == 1 { + println!( + "Splitting and uploading {:?} into {chunks_to_upload_len} chunks", + self.file_paths_to_print[0] + ); + } else { + println!( + "Splitting and uploading {:?} into {chunks_to_upload_len} chunks", + self.file_paths_to_print + ); + } + } + + fn on_upload_complete( + &self, + upload_sum: &UploadSummary, + elapsed_time: Duration, + chunks_to_upload_len: usize, + ) { let elapsed_minutes = elapsed_time.as_secs() / 60; let elapsed_seconds = elapsed_time.as_secs() % 60; - if elapsed_minutes > 0 { + let elapsed = if elapsed_minutes > 0 { format!("{elapsed_minutes} minutes {elapsed_seconds} seconds") } else { format!("{elapsed_seconds} seconds") - } - } + }; - fn print_uploaded_msg(make_data_public: bool) { + println!( + "Among {chunks_to_upload_len} chunks, found {} already existed in network, uploaded \ + the leftover {} chunks in {elapsed}", + upload_sum.skipped_count, upload_sum.uploaded_count, + ); + info!( + "Among {chunks_to_upload_len} chunks, found {} already existed in network, uploaded \ + the leftover {} chunks in {elapsed}", + upload_sum.skipped_count, upload_sum.uploaded_count, + ); println!("**************************************"); - println!("* Uploaded Files *"); - if !make_data_public { - println!("* *"); - println!("* These are not public by default. *"); - println!("* Reupload with `-p` option *"); - println!("* to publish the datamaps. *"); - } + println!("* Payment Details *"); println!("**************************************"); + println!( + "Made payment of {:?} for {} chunks", + upload_sum.storage_cost, upload_sum.uploaded_count + ); + println!( + "Made payment of {:?} for royalties fees", + upload_sum.royalty_fees + ); + println!("New wallet balance: {}", upload_sum.final_balance); } +} - fn print_completed_file_list(chunk_manager: &ChunkManager) { - for (_, file_name, addr) in chunk_manager.completed_files() { +impl StdOutPrinter { + fn print_completed_file_list(completed_files: &[(PathBuf, OsString, ChunkAddress)]) { + for (_, file_name, addr) in completed_files { let hex_addr = addr.to_hex(); if let Some(file_name) = file_name.to_str() { println!("\"{file_name}\" {hex_addr}"); @@ -355,4 +470,16 @@ impl FilesUploader { } } } + + fn print_uploaded_msg(make_data_public: bool) { + println!("**************************************"); + println!("* Uploaded Files *"); + if !make_data_public { + println!("* *"); + println!("* These are not public by default. *"); + println!("* Reupload with `-p` option *"); + println!("* to publish the datamaps. *"); + } + println!("**************************************"); + } } diff --git a/sn_cli/src/lib.rs b/sn_cli/src/lib.rs index 637f8a1581..5d39b650c5 100644 --- a/sn_cli/src/lib.rs +++ b/sn_cli/src/lib.rs @@ -11,6 +11,6 @@ mod files; pub use acc_packet::AccountPacket; pub use files::{ - download_file, download_files, ChunkManager, Estimator, FilesUploader, UploadedFile, - UPLOADED_FILES, + download_file, download_files, ChunkManager, Estimator, FilesUploadStatusNotifier, + FilesUploader, UploadedFile, UPLOADED_FILES, }; From ec62653f134b3d7d8341bb5b284c14a034fe9d24 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 9 Apr 2024 14:36:39 +0530 Subject: [PATCH 030/205] fix(protocol): get proper version str for pre releases --- sn_protocol/src/version.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sn_protocol/src/version.rs b/sn_protocol/src/version.rs index 3c5074c1cc..0f1b9a7e40 100644 --- a/sn_protocol/src/version.rs +++ b/sn_protocol/src/version.rs @@ -75,12 +75,13 @@ fn write_network_version_with_slash() -> String { } // Protocol support shall be downward compatible for patch only version update. -// i.e. versions of `A.B.X` shall be considered as a same protocol of `A.B` -// And any pre-release versions `A.B.C-alpha.X` shall be considered as a same protocol of `A.B.C-alpha` -fn get_truncate_version_str() -> &'static str { +// i.e. versions of `A.B.X` or `A.B.X-alpha.Y` shall be considered as a same protocol of `A.B` +fn get_truncate_version_str() -> String { let version_str = env!("CARGO_PKG_VERSION"); - match version_str.rfind('.') { - Some(pos) => &version_str[..pos], - None => version_str, + let parts = version_str.split('.').collect::>(); + if parts.len() >= 2 { + format!("{}.{}", parts[0], parts[1]) + } else { + panic!("Cannot obtain truncated version str for {version_str:?}: {parts:?}"); } } From 6d905e9e3d8166634bcc03493001d082979aa505 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Fri, 5 Apr 2024 15:19:27 +0900 Subject: [PATCH 031/205] fix(networking): increase the local responsible range of nodes to K_VALUE peers away --- sn_networking/src/driver.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index b91ddfa112..978e2eaa95 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -10,6 +10,7 @@ use crate::metrics::NetworkMetrics; #[cfg(feature = "open-metrics")] use crate::metrics_service::run_metrics_server; +use crate::NodeIssue; use crate::{ bootstrap::{ContinuousBootstrap, BOOTSTRAP_INTERVAL}, circular_vec::CircularVec, @@ -26,7 +27,6 @@ use crate::{ target_arch::{interval, spawn, Instant}, transport, Network, CLOSE_GROUP_SIZE, }; -use crate::{NodeIssue, REPLICATE_RANGE}; use futures::StreamExt; use libp2p::kad::KBucketDistance as Distance; #[cfg(feature = "local-discovery")] @@ -606,7 +606,7 @@ impl SwarmDriver { if !self.is_client { let closest_k_peers = self.get_closest_k_value_local_peers(); - if let Some(distance) = self.get_farthest_relevant_address_estimate(&closest_k_peers) { + if let Some(distance) = self.get_farthest_data_address_estimate(&closest_k_peers) { // set any new distance to farthest record in the store self.swarm.behaviour_mut().kademlia.store_mut().set_distance_range(distance); // the distance range within the replication_fetcher shall be in sync as well @@ -624,8 +624,8 @@ impl SwarmDriver { /// Return a far address, close to but probably farther than our responsibilty range. /// This simply uses the closest k peers to estimate the farthest address as - /// `REPLICATE_RANGE`th peer's address distance. - fn get_farthest_relevant_address_estimate( + /// `K_VALUE`th peer's address distance. + fn get_farthest_data_address_estimate( &mut self, // Sorted list of closest k peers to our peer id. closest_k_peers: &[PeerId], @@ -635,11 +635,11 @@ impl SwarmDriver { let our_address = NetworkAddress::from_peer(self.self_peer_id); - // get REPLICATE_RANGE + 2 peer's address distance + // get K_VALUE/2 peer's address distance // This is a rough estimate of the farthest address we might be responsible for. // We want this to be higher than actually necessary, so we retain more data // and can be sure to pass bad node checks - if let Some(peer) = closest_k_peers.get(REPLICATE_RANGE + 2) { + if let Some(peer) = closest_k_peers.last() { let address = NetworkAddress::from_peer(*peer); let distance = our_address.distance(&address); farthest_distance = Some(distance); From 17d292b2c2a2700a38b2a68ed69833c8ac81985a Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Mon, 8 Apr 2024 10:51:02 +0900 Subject: [PATCH 032/205] feat(networking): shift to use ilog2 bucket distance for close data calcs ilog2 is about magnitude rather than specifics as distance is already being estimated we can use this to easily render a buffer zone of data we replicate vs a close bucket which we deem to be our responsibility --- sn_networking/src/cmd.rs | 9 ++-- sn_networking/src/driver.rs | 16 ++++---- sn_networking/src/event.rs | 6 +-- sn_networking/src/lib.rs | 4 +- sn_networking/src/record_store.rs | 52 ++++++++++++++---------- sn_networking/src/record_store_api.rs | 10 ++--- sn_networking/src/replication_fetcher.rs | 17 ++++---- sn_node/src/replication.rs | 4 +- 8 files changed, 65 insertions(+), 53 deletions(-) diff --git a/sn_networking/src/cmd.rs b/sn_networking/src/cmd.rs index 355d3d276d..ec7d6bea75 100644 --- a/sn_networking/src/cmd.rs +++ b/sn_networking/src/cmd.rs @@ -10,7 +10,7 @@ use crate::{ driver::{PendingGetClosestType, SwarmDriver}, error::{NetworkError, Result}, multiaddr_pop_p2p, GetRecordCfg, GetRecordError, MsgResponder, NetworkEvent, CLOSE_GROUP_SIZE, - REPLICATE_RANGE, + REPLICATION_PEERS_COUNT, }; use libp2p::{ kad::{store::RecordStore, Quorum, Record, RecordKey}, @@ -479,9 +479,10 @@ impl SwarmDriver { .behaviour_mut() .kademlia .store_mut() - .get_distance_range() + .get_farthest_replication_distance_bucket() { - self.replication_fetcher.set_distance_range(distance); + self.replication_fetcher + .set_replication_distance_range(distance); } if let Err(err) = result { @@ -818,7 +819,7 @@ impl SwarmDriver { let replicate_targets = closest_k_peers .into_iter() // add some leeway to allow for divergent knowledge - .take(REPLICATE_RANGE) + .take(REPLICATION_PEERS_COUNT) .collect::>(); let all_records: Vec<_> = self diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index 978e2eaa95..c50c848568 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -28,7 +28,6 @@ use crate::{ transport, Network, CLOSE_GROUP_SIZE, }; use futures::StreamExt; -use libp2p::kad::KBucketDistance as Distance; #[cfg(feature = "local-discovery")] use libp2p::mdns; use libp2p::{ @@ -609,8 +608,10 @@ impl SwarmDriver { if let Some(distance) = self.get_farthest_data_address_estimate(&closest_k_peers) { // set any new distance to farthest record in the store self.swarm.behaviour_mut().kademlia.store_mut().set_distance_range(distance); + + let replication_distance = self.swarm.behaviour_mut().kademlia.store_mut().get_farthest_replication_distance_bucket().unwrap_or(1); // the distance range within the replication_fetcher shall be in sync as well - self.replication_fetcher.set_distance_range(distance); + self.replication_fetcher.set_replication_distance_range(replication_distance); } } } @@ -622,27 +623,26 @@ impl SwarmDriver { // ---------- Crate helpers ------------------- // -------------------------------------------- - /// Return a far address, close to but probably farther than our responsibilty range. + /// Returns the farthest bucket, close to but probably farther than our responsibilty range. /// This simply uses the closest k peers to estimate the farthest address as - /// `K_VALUE`th peer's address distance. + /// `K_VALUE`th peer's bucket. fn get_farthest_data_address_estimate( &mut self, // Sorted list of closest k peers to our peer id. closest_k_peers: &[PeerId], - ) -> Option { + ) -> Option { // if we don't have enough peers we don't set the distance range yet. let mut farthest_distance = None; let our_address = NetworkAddress::from_peer(self.self_peer_id); - // get K_VALUE/2 peer's address distance + // get K_VALUEth peer's address distance // This is a rough estimate of the farthest address we might be responsible for. // We want this to be higher than actually necessary, so we retain more data // and can be sure to pass bad node checks if let Some(peer) = closest_k_peers.last() { let address = NetworkAddress::from_peer(*peer); - let distance = our_address.distance(&address); - farthest_distance = Some(distance); + farthest_distance = our_address.distance(&address).ilog2(); } farthest_distance diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 45c4d95b7f..2cde322462 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -12,7 +12,7 @@ use crate::{ error::{NetworkError, Result}, multiaddr_is_global, multiaddr_strip_p2p, sort_peers_by_address, target_arch::Instant, - CLOSE_GROUP_SIZE, REPLICATE_RANGE, + CLOSE_GROUP_SIZE, REPLICATION_PEERS_COUNT, }; use core::fmt; use custom_debug::Debug as CustomDebug; @@ -1310,12 +1310,12 @@ impl SwarmDriver { target: &NetworkAddress, all_peers: &Vec, ) -> bool { - if all_peers.len() <= REPLICATE_RANGE { + if all_peers.len() <= REPLICATION_PEERS_COUNT { return true; } // Margin of 2 to allow our RT being bit lagging. - match sort_peers_by_address(all_peers, target, REPLICATE_RANGE) { + match sort_peers_by_address(all_peers, target, REPLICATION_PEERS_COUNT) { Ok(close_group) => close_group.contains(&our_peer_id), Err(err) => { warn!("Could not get sorted peers for {target:?} with error {err:?}"); diff --git a/sn_networking/src/lib.rs b/sn_networking/src/lib.rs index 31a6d4523f..62046b2484 100644 --- a/sn_networking/src/lib.rs +++ b/sn_networking/src/lib.rs @@ -82,9 +82,9 @@ pub type PayeeQuote = (PeerId, MainPubkey, PaymentQuote); /// The size has been set to 5 for improved performance. pub const CLOSE_GROUP_SIZE: usize = 5; -/// The range of peers that will be considered as close to a record target, +/// The count of peers that will be considered as close to a record target, /// that a replication of the record shall be sent/accepted to/by the peer. -pub const REPLICATE_RANGE: usize = CLOSE_GROUP_SIZE + 2; +pub const REPLICATION_PEERS_COUNT: usize = CLOSE_GROUP_SIZE + 2; /// Majority of a given group (i.e. > 1/2). #[inline] diff --git a/sn_networking/src/record_store.rs b/sn_networking/src/record_store.rs index c07579d1e7..deb19abc27 100644 --- a/sn_networking/src/record_store.rs +++ b/sn_networking/src/record_store.rs @@ -19,7 +19,7 @@ use libp2p::{ identity::PeerId, kad::{ store::{Error, RecordStore, Result}, - KBucketDistance as Distance, KBucketKey, ProviderRecord, Record, RecordKey as Key, + KBucketKey, ProviderRecord, Record, RecordKey as Key, }, }; #[cfg(feature = "open-metrics")] @@ -58,9 +58,10 @@ pub struct NodeRecordStore { network_event_sender: mpsc::Sender, /// Send cmds to the network layer. Used to interact with self in an async fashion. swarm_cmd_sender: mpsc::Sender, - /// Distance range specify the acceptable range of record entry. + /// ilog2 distance range of responsible records + /// AKA: how many buckets of data do we consider "close" /// None means accept all records. - distance_range: Option, + responsible_distance_range: Option, #[cfg(feature = "open-metrics")] /// Used to report the number of records held by the store to the metrics server. record_count_metric: Option, @@ -195,7 +196,7 @@ impl NodeRecordStore { records, network_event_sender, swarm_cmd_sender, - distance_range: None, + responsible_distance_range: None, #[cfg(feature = "open-metrics")] record_count_metric: None, received_payment_count: 0, @@ -211,9 +212,9 @@ impl NodeRecordStore { self } - /// Returns the current distance range - pub fn get_distance_range(&self) -> Option { - self.distance_range + /// Returns the current distance ilog2 (aka bucket) range of CLOSE_GROUP nodes. + pub fn get_responsible_distance_range(&self) -> Option { + self.responsible_distance_range } // Converts a Key into a Hex string. @@ -438,7 +439,7 @@ impl NodeRecordStore { live_time: self.timestamp.elapsed().as_secs(), }; - if let Some(distance_range) = self.distance_range { + if let Some(distance_range) = self.responsible_distance_range { let relevant_records = self.get_records_within_distance_range(record_keys_as_hashset, distance_range); @@ -474,7 +475,7 @@ impl NodeRecordStore { pub fn get_records_within_distance_range( &self, records: HashSet<&Key>, - distance_range: Distance, + distance_range: u32, ) -> usize { debug!( "Total record count is {:?}. Distance is: {distance_range:?}", @@ -485,7 +486,7 @@ impl NodeRecordStore { .iter() .filter(|key| { let kbucket_key = KBucketKey::new(key.to_vec()); - distance_range >= self.local_key.distance(&kbucket_key) + distance_range >= self.local_key.distance(&kbucket_key).ilog2().unwrap_or(0) }) .count(); @@ -494,8 +495,8 @@ impl NodeRecordStore { } /// Setup the distance range. - pub(crate) fn set_distance_range(&mut self, distance_range: Distance) { - self.distance_range = Some(distance_range); + pub(crate) fn set_responsible_distance_range(&mut self, farthest_responsible_bucket: u32) { + self.responsible_distance_range = Some(farthest_responsible_bucket); } } @@ -730,7 +731,7 @@ pub fn calculate_cost_for_records(quoting_metrics: &QuotingMetrics) -> u64 { mod tests { use super::*; - use crate::{close_group_majority, sort_peers_by_key, REPLICATE_RANGE}; + use crate::{close_group_majority, sort_peers_by_key, REPLICATION_PEERS_COUNT}; use bytes::Bytes; use eyre::ContextCompat; use libp2p::{core::multihash::Multihash, kad::RecordKey}; @@ -1076,7 +1077,7 @@ mod tests { #[tokio::test] #[allow(clippy::mutable_key_type)] - async fn get_records_within_distance_range() -> eyre::Result<()> { + async fn get_records_within_bucket_range() -> eyre::Result<()> { let max_records = 50; let temp_dir = std::env::temp_dir(); @@ -1140,16 +1141,21 @@ mod tests { .wrap_err("Could not parse record store key")?, ); // get the distance to this record from our local key - let distance = self_address.distance(&halfway_record_address); + let distance = self_address + .distance(&halfway_record_address) + .ilog2() + .unwrap_or(0); - store.set_distance_range(distance); + // must be plus one bucket from the halfway record + store.set_responsible_distance_range(distance); let record_keys = store.records.keys().collect(); - // check that the number of records returned is correct - assert_eq!( - store.get_records_within_distance_range(record_keys, distance), - stored_records.len() / 2 + // check that the number of records returned is larger than half our records + // (ie, that we cover _at least_ all the records within our distance range) + assert!( + store.get_records_within_distance_range(record_keys, distance) + >= stored_records.len() / 2 ); Ok(()) @@ -1177,7 +1183,11 @@ mod tests { for _ in 0..num_of_chunks_per_itr { let name = xor_name::rand::random(); let address = NetworkAddress::from_chunk_address(ChunkAddress::new(name)); - match sort_peers_by_key(&peers_vec, &address.as_kbucket_key(), REPLICATE_RANGE) { + match sort_peers_by_key( + &peers_vec, + &address.as_kbucket_key(), + REPLICATION_PEERS_COUNT, + ) { Ok(peers_in_replicate_range) => { let peers_in_replicate_range: Vec = peers_in_replicate_range .iter() diff --git a/sn_networking/src/record_store_api.rs b/sn_networking/src/record_store_api.rs index c04ab564b8..08d7074eaa 100644 --- a/sn_networking/src/record_store_api.rs +++ b/sn_networking/src/record_store_api.rs @@ -10,7 +10,7 @@ use crate::record_store::{ClientRecordStore, NodeRecordStore}; use libp2p::kad::{ store::{RecordStore, Result}, - KBucketDistance as Distance, ProviderRecord, Record, RecordKey, + ProviderRecord, Record, RecordKey, }; use sn_protocol::{storage::RecordType, NetworkAddress}; use sn_transfers::{NanoTokens, QuotingMetrics}; @@ -131,22 +131,22 @@ impl UnifiedRecordStore { } } - pub(crate) fn get_distance_range(&self) -> Option { + pub(crate) fn get_farthest_replication_distance_bucket(&self) -> Option { match self { Self::Client(_store) => { warn!("Calling get_distance_range at Client. This should not happen"); None } - Self::Node(store) => store.get_distance_range(), + Self::Node(store) => store.get_responsible_distance_range(), } } - pub(crate) fn set_distance_range(&mut self, distance: Distance) { + pub(crate) fn set_distance_range(&mut self, distance: u32) { match self { Self::Client(_store) => { warn!("Calling set_distance_range at Client. This should not happen"); } - Self::Node(store) => store.set_distance_range(distance), + Self::Node(store) => store.set_responsible_distance_range(distance), } } diff --git a/sn_networking/src/replication_fetcher.rs b/sn_networking/src/replication_fetcher.rs index c3188bc5d1..046b3d693c 100644 --- a/sn_networking/src/replication_fetcher.rs +++ b/sn_networking/src/replication_fetcher.rs @@ -10,7 +10,7 @@ use crate::target_arch::spawn; use crate::{event::NetworkEvent, target_arch::Instant}; use libp2p::{ - kad::{KBucketDistance as Distance, RecordKey, K_VALUE}, + kad::{RecordKey, K_VALUE}, PeerId, }; use sn_protocol::{storage::RecordType, NetworkAddress, PrettyPrintRecordKey}; @@ -41,8 +41,8 @@ pub(crate) struct ReplicationFetcher { // Avoid fetching same chunk from different nodes AND carry out too many parallel tasks. on_going_fetches: HashMap<(RecordKey, RecordType), (PeerId, ReplicationTimeout)>, event_sender: mpsc::Sender, - // Distance range that the incoming key shall be fetched - distance_range: Option, + /// ilog2 bucket distance range that the incoming key shall be fetched + distance_range: Option, } impl ReplicationFetcher { @@ -58,7 +58,7 @@ impl ReplicationFetcher { } /// Set the distance range. - pub(crate) fn set_distance_range(&mut self, distance_range: Distance) { + pub(crate) fn set_replication_distance_range(&mut self, distance_range: u32) { self.distance_range = Some(distance_range); } @@ -102,7 +102,8 @@ impl ReplicationFetcher { let self_address = NetworkAddress::from_peer(self.self_peer_id); incoming_keys.retain(|(addr, _record_type)| { - let is_in_range = self_address.distance(addr) <= *distance_range; + let is_in_range = + self_address.distance(addr).ilog2().unwrap_or(0) <= *distance_range; if !is_in_range { out_of_range_keys.push(addr.clone()); } @@ -368,8 +369,8 @@ mod tests { // Set distance range let distance_target = NetworkAddress::from_peer(PeerId::random()); - let distance_range = self_address.distance(&distance_target); - replication_fetcher.set_distance_range(distance_range); + let distance_range = self_address.distance(&distance_target).ilog2().unwrap_or(1); + replication_fetcher.set_replication_distance_range(distance_range); let mut incoming_keys = Vec::new(); let mut in_range_keys = 0; @@ -377,7 +378,7 @@ mod tests { let random_data: Vec = (0..50).map(|_| rand::random::()).collect(); let key = NetworkAddress::from_record_key(&RecordKey::from(random_data)); - if key.distance(&self_address) <= distance_range { + if key.distance(&self_address).ilog2().unwrap_or(0) <= distance_range { in_range_keys += 1; } diff --git a/sn_node/src/replication.rs b/sn_node/src/replication.rs index 8d500e0ee0..4de65952ff 100644 --- a/sn_node/src/replication.rs +++ b/sn_node/src/replication.rs @@ -11,7 +11,7 @@ use libp2p::{ kad::{Quorum, Record, RecordKey}, PeerId, }; -use sn_networking::{sort_peers_by_address, GetRecordCfg, Network, REPLICATE_RANGE}; +use sn_networking::{sort_peers_by_address, GetRecordCfg, Network, REPLICATION_PEERS_COUNT}; use sn_protocol::{ messages::{Cmd, Query, QueryResponse, Request, Response}, storage::RecordType, @@ -150,7 +150,7 @@ impl Node { let sorted_based_on_addr = match sort_peers_by_address( &closest_k_peers, &data_addr, - REPLICATE_RANGE, + REPLICATION_PEERS_COUNT, ) { Ok(result) => result, Err(err) => { From db8098cac9219ee7748cf5d07daeb9d2223c77f1 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Tue, 9 Apr 2024 19:49:41 +0100 Subject: [PATCH 033/205] fix: incorrect release type reference Already tried to fix this bug with the faucet upgrade process downloading the wrong, but somehow still managed to miss another hard-coded `ReleaseType` reference. --- sn_node_manager/src/cmd/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sn_node_manager/src/cmd/mod.rs b/sn_node_manager/src/cmd/mod.rs index 02a3e5a93f..4634d3b09d 100644 --- a/sn_node_manager/src/cmd/mod.rs +++ b/sn_node_manager/src/cmd/mod.rs @@ -51,7 +51,7 @@ pub async fn download_and_get_upgrade_bin_path( let latest_version = release_repo.get_latest_version(&release_type).await?; println!("Latest version is {latest_version}"); let (upgrade_bin_path, _) = download_and_extract_release( - ReleaseType::Safenode, + release_type, None, Some(latest_version.to_string()), &*release_repo, From 5cd1a19fefb649b7011f1c370ce81fb4a24f8787 Mon Sep 17 00:00:00 2001 From: qima Date: Tue, 9 Apr 2024 23:13:04 +0800 Subject: [PATCH 034/205] feat(node): restore historic quoting metrics to allow restart --- sn_networking/src/record_store.rs | 83 ++++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 7 deletions(-) diff --git a/sn_networking/src/record_store.rs b/sn_networking/src/record_store.rs index deb19abc27..d5cb5e8b15 100644 --- a/sn_networking/src/record_store.rs +++ b/sn_networking/src/record_store.rs @@ -25,6 +25,7 @@ use libp2p::{ #[cfg(feature = "open-metrics")] use prometheus_client::metrics::gauge::Gauge; use rand::RngCore; +use serde::{Deserialize, Serialize}; use sn_protocol::{ storage::{RecordHeader, RecordKind, RecordType}, NetworkAddress, PrettyPrintRecordKey, @@ -35,6 +36,7 @@ use std::{ collections::{HashMap, HashSet}, fs, path::{Path, PathBuf}, + time::SystemTime, vec, }; use tokio::sync::mpsc; @@ -44,6 +46,9 @@ use xor_name::XorName; /// Max number of records a node can store const MAX_RECORDS_COUNT: usize = 2048; +/// File name of the recorded historical quoting metrics. +const HISTORICAL_QUOTING_METRICS_FILENAME: &str = "historic_quoting_metrics"; + /// A `RecordStore` that stores records on disk. pub struct NodeRecordStore { /// The identity of the peer owning the store. @@ -71,7 +76,7 @@ pub struct NodeRecordStore { /// Plus a 4 byte nonce starter encryption_details: (Aes256GcmSiv, [u8; 4]), /// Time that this record_store got started - timestamp: Instant, + timestamp: SystemTime, } /// Configuration for a `DiskBackedRecordStore`. @@ -105,9 +110,15 @@ fn generate_nonce_for_record(nonce_starter: &[u8; 4], key: &Key) -> Nonce { Nonce::from_iter(nonce_bytes) } +#[derive(Clone, Serialize, Deserialize)] +struct HistoricQuotingMetrics { + received_payment_count: usize, + timestamp: SystemTime, +} + impl NodeRecordStore { /// If a directory for our node already exists, repopulate the records from the files in the dir - pub fn update_records_from_an_existing_store( + fn update_records_from_an_existing_store( config: &NodeRecordStoreConfig, encryption_details: &(Aes256GcmSiv, [u8; 4]), ) -> HashMap { @@ -175,6 +186,38 @@ impl NodeRecordStore { records } + /// If quote_metrics file already exists, using the existing parameters. + fn restore_quoting_metrics(storage_dir: &Path) -> Option { + let file_path = storage_dir.join(HISTORICAL_QUOTING_METRICS_FILENAME); + + if let Ok(file) = fs::File::open(file_path) { + if let Ok(quoting_metrics) = rmp_serde::from_read(&file) { + return Some(quoting_metrics); + } + } + + None + } + + fn flush_historic_quoting_metrics(&self) { + let file_path = self + .config + .storage_dir + .join(HISTORICAL_QUOTING_METRICS_FILENAME); + + let historic_quoting_metrics = HistoricQuotingMetrics { + received_payment_count: self.received_payment_count, + timestamp: self.timestamp, + }; + + spawn(async move { + if let Ok(mut file) = fs::File::create(file_path) { + let mut serialiser = rmp_serde::encode::Serializer::new(&mut file); + let _ = historic_quoting_metrics.serialize(&mut serialiser); + } + }); + } + /// Creates a new `DiskBackedStore` with the given configuration. pub fn with_config( local_id: PeerId, @@ -188,8 +231,22 @@ impl NodeRecordStore { OsRng.fill_bytes(&mut nonce_starter); let encryption_details = (cipher, nonce_starter); + + // Recover the quoting_metrics first, as the historical file will be cleaned by + // the later on update_records_from_an_existing_store function + let (received_payment_count, timestamp) = if let Some(historic_quoting_metrics) = + Self::restore_quoting_metrics(&config.storage_dir) + { + ( + historic_quoting_metrics.received_payment_count, + historic_quoting_metrics.timestamp, + ) + } else { + (0, SystemTime::now()) + }; + let records = Self::update_records_from_an_existing_store(&config, &encryption_details); - NodeRecordStore { + let record_store = NodeRecordStore { local_key: KBucketKey::from(local_id), local_address: NetworkAddress::from_peer(local_id), config, @@ -199,10 +256,14 @@ impl NodeRecordStore { responsible_distance_range: None, #[cfg(feature = "open-metrics")] record_count_metric: None, - received_payment_count: 0, + received_payment_count, encryption_details, - timestamp: Instant::now(), - } + timestamp, + }; + + record_store.flush_historic_quoting_metrics(); + + record_store } /// Set the record_count_metric to report the number of records stored to the metrics server @@ -432,11 +493,17 @@ impl NodeRecordStore { let records_stored = self.records.len(); let record_keys_as_hashset: HashSet<&Key> = self.records.keys().collect(); + let live_time = if let Ok(elapsed) = self.timestamp.elapsed() { + elapsed.as_secs() + } else { + 0 + }; + let mut quoting_metrics = QuotingMetrics { close_records_stored: records_stored, max_records: self.config.max_records, received_payment_count: self.received_payment_count, - live_time: self.timestamp.elapsed().as_secs(), + live_time, }; if let Some(distance_range) = self.responsible_distance_range { @@ -468,6 +535,8 @@ impl NodeRecordStore { /// Notify the node received a payment. pub(crate) fn payment_received(&mut self) { self.received_payment_count = self.received_payment_count.saturating_add(1); + + self.flush_historic_quoting_metrics(); } /// Calculate how many records are stored within a distance range From 40b11f98cf6acb40d00c8d5eb5e3d79e6ea5481f Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Wed, 10 Apr 2024 12:54:35 +0900 Subject: [PATCH 035/205] feat(faucet): log from sn_client --- sn_client/src/wallet.rs | 1 + sn_faucet/src/main.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/sn_client/src/wallet.rs b/sn_client/src/wallet.rs index 04b963fb8e..3ece569a77 100644 --- a/sn_client/src/wallet.rs +++ b/sn_client/src/wallet.rs @@ -1053,6 +1053,7 @@ pub async fn send( .await { println!("Wallet has pre-unconfirmed transactions, can't progress further."); + warn!("Wallet has pre-unconfirmed transactions, can't progress further."); return Err(err.into()); } diff --git a/sn_faucet/src/main.rs b/sn_faucet/src/main.rs index 935d0066f2..07ac8d54a2 100644 --- a/sn_faucet/src/main.rs +++ b/sn_faucet/src/main.rs @@ -41,6 +41,7 @@ async fn main() -> Result<()> { let logging_targets = vec![ // TODO: Reset to nice and clean defaults once we have a better idea of what we want ("faucet".to_string(), Level::TRACE), + ("sn_client".to_string(), Level::TRACE), ("sn_faucet".to_string(), Level::TRACE), ("sn_networking".to_string(), Level::DEBUG), ("sn_build_info".to_string(), Level::TRACE), From c3f724621e58bfda3394c234dadd257f24e6a832 Mon Sep 17 00:00:00 2001 From: Mark Date: Tue, 9 Apr 2024 18:04:29 +0100 Subject: [PATCH 036/205] chore(cli): make FilesUploadSummary public --- sn_cli/src/files.rs | 2 +- sn_cli/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sn_cli/src/files.rs b/sn_cli/src/files.rs index 79b12cfc22..66341f4865 100644 --- a/sn_cli/src/files.rs +++ b/sn_cli/src/files.rs @@ -15,7 +15,7 @@ mod upload; pub use chunk_manager::ChunkManager; pub use download::{download_file, download_files}; pub use estimate::Estimator; -pub use files_uploader::{FilesUploadStatusNotifier, FilesUploader}; +pub use files_uploader::{FilesUploadStatusNotifier, FilesUploadSummary, FilesUploader}; pub use upload::{UploadedFile, UPLOADED_FILES}; use color_eyre::Result; diff --git a/sn_cli/src/lib.rs b/sn_cli/src/lib.rs index 5d39b650c5..0a85ce69b3 100644 --- a/sn_cli/src/lib.rs +++ b/sn_cli/src/lib.rs @@ -12,5 +12,5 @@ mod files; pub use acc_packet::AccountPacket; pub use files::{ download_file, download_files, ChunkManager, Estimator, FilesUploadStatusNotifier, - FilesUploader, UploadedFile, UPLOADED_FILES, + FilesUploadSummary, FilesUploader, UploadedFile, UPLOADED_FILES, }; From a0e98ec1a58b5d8b7c95314f6c9f5f8d5530d86b Mon Sep 17 00:00:00 2001 From: qima Date: Mon, 8 Apr 2024 18:53:57 +0800 Subject: [PATCH 037/205] feat(node): notify peer it is now considered as BAD --- sn_networking/src/cmd.rs | 13 ++++++++++ sn_networking/src/event.rs | 38 ++++++++++++++++++++++++++++ sn_node/src/node.rs | 19 +++++++++++++- sn_protocol/src/messages/cmd.rs | 26 +++++++++++++++++++ sn_protocol/src/messages/response.rs | 5 ++++ 5 files changed, 100 insertions(+), 1 deletion(-) diff --git a/sn_networking/src/cmd.rs b/sn_networking/src/cmd.rs index ec7d6bea75..87c18dd2c2 100644 --- a/sn_networking/src/cmd.rs +++ b/sn_networking/src/cmd.rs @@ -735,6 +735,9 @@ impl SwarmDriver { info!("Peer {peer_id:?} is reported as having issue {issue:?}"); let (issue_vec, is_bad) = self.bad_nodes.entry(peer_id).or_default(); + let mut is_new_bad = false; + let mut bad_behaviour: String = "".to_string(); + // If being considered as bad already, skip certain operations if !(*is_bad) { // Remove outdated entries @@ -767,6 +770,8 @@ impl SwarmDriver { .count(); if issue_counts >= 3 { *is_bad = true; + is_new_bad = true; + bad_behaviour = format!("{issue:?}"); info!("Peer {peer_id:?} accumulated {issue_counts} times of issue {issue:?}. Consider it as a bad node now."); // Once a bad behaviour detected, no point to continue break; @@ -785,6 +790,14 @@ impl SwarmDriver { self.log_kbuckets(&peer_id); let _ = self.check_for_change_in_our_close_group(); } + + if is_new_bad { + self.send_event(NetworkEvent::PeerConsideredAsBad { + detected_by: self.self_peer_id, + bad_peer: peer_id, + bad_behaviour, + }); + } } } diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 2cde322462..9c777bb8ff 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -114,6 +114,12 @@ pub enum NetworkEvent { our_protocol: String, their_protocol: String, }, + /// The peer is now considered as a bad node, due to the detected bad behaviour + PeerConsideredAsBad { + detected_by: PeerId, + bad_peer: PeerId, + bad_behaviour: String, + }, /// The records bearing these keys are to be fetched from the holder or the network KeysToFetchForReplication(Vec<(PeerId, RecordKey)>), /// Started listening on a new address @@ -160,6 +166,16 @@ impl Debug for NetworkEvent { } => { write!(f, "NetworkEvent::PeerWithUnsupportedProtocol({our_protocol:?}, {their_protocol:?})") } + NetworkEvent::PeerConsideredAsBad { + bad_peer, + bad_behaviour, + .. + } => { + write!( + f, + "NetworkEvent::PeerConsideredAsBad({bad_peer:?}, {bad_behaviour:?})" + ) + } NetworkEvent::KeysToFetchForReplication(list) => { let keys_len = list.len(); write!(f, "NetworkEvent::KeysForReplication({keys_len:?})") @@ -695,6 +711,28 @@ impl SwarmDriver { .collect(); self.send_event(NetworkEvent::QuoteVerification { quotes }) } + Request::Cmd(sn_protocol::messages::Cmd::PeerConsideredAsBad { + detected_by, + bad_peer, + bad_behaviour, + }) => { + let response = Response::Cmd( + sn_protocol::messages::CmdResponse::PeerConsideredAsBad(Ok(())), + ); + self.swarm + .behaviour_mut() + .request_response + .send_response(channel, response) + .map_err(|_| NetworkError::InternalMsgChannelDropped)?; + + if bad_peer == NetworkAddress::from_peer(self.self_peer_id) { + warn!("Peer {detected_by:?} consider us as BAD, due to {bad_behaviour:?}."); + // TODO: shall we terminate self after received such notifications + // from the majority close_group nodes around us? + } else { + error!("Received a bad_peer notification from {detected_by:?}, targeting {bad_peer:?}, which is not us."); + } + } Request::Query(query) => { self.send_event(NetworkEvent::QueryRequestReceived { query, diff --git a/sn_node/src/node.rs b/sn_node/src/node.rs index 0ee4b8ffe2..85c21942f4 100644 --- a/sn_node/src/node.rs +++ b/sn_node/src/node.rs @@ -27,7 +27,7 @@ use sn_networking::{ }; use sn_protocol::{ error::Error as ProtocolError, - messages::{ChunkProof, CmdResponse, Query, QueryResponse, Request, Response}, + messages::{ChunkProof, Cmd, CmdResponse, Query, QueryResponse, Request, Response}, NetworkAddress, PrettyPrintRecordKey, }; use sn_transfers::{HotWallet, MainPubkey, MainSecretKey, NanoTokens}; @@ -329,6 +329,23 @@ impl Node { NetworkEvent::PeerWithUnsupportedProtocol { .. } => { event_header = "PeerWithUnsupportedProtocol"; } + NetworkEvent::PeerConsideredAsBad { + detected_by, + bad_peer, + bad_behaviour, + } => { + event_header = "PeerConsideredAsBad"; + let request = Request::Cmd(Cmd::PeerConsideredAsBad { + detected_by: NetworkAddress::from_peer(detected_by), + bad_peer: NetworkAddress::from_peer(bad_peer), + bad_behaviour, + }); + + let network = self.network.clone(); + let _handle = spawn(async move { + network.send_req_ignore_reply(request, bad_peer); + }); + } NetworkEvent::NewListenAddr(_) => { event_header = "NewListenAddr"; if !cfg!(feature = "local-discovery") { diff --git a/sn_protocol/src/messages/cmd.rs b/sn_protocol/src/messages/cmd.rs index 31222399e7..7b62fce164 100644 --- a/sn_protocol/src/messages/cmd.rs +++ b/sn_protocol/src/messages/cmd.rs @@ -35,6 +35,12 @@ pub enum Cmd { target: NetworkAddress, quotes: Vec<(NetworkAddress, PaymentQuote)>, }, + /// Notify the peer it is now being considered as BAD due to the included behaviour + PeerConsideredAsBad { + detected_by: NetworkAddress, + bad_peer: NetworkAddress, + bad_behaviour: String, + }, } impl std::fmt::Debug for Cmd { @@ -53,6 +59,16 @@ impl std::fmt::Debug for Cmd { .field("target", target) .field("quotes_len", "es.len()) .finish(), + Cmd::PeerConsideredAsBad { + detected_by, + bad_peer, + bad_behaviour, + } => f + .debug_struct("Cmd::PeerConsideredAsBad") + .field("detected_by", detected_by) + .field("bad_peer", bad_peer) + .field("bad_behaviour", bad_behaviour) + .finish(), } } } @@ -63,6 +79,7 @@ impl Cmd { match self { Cmd::Replicate { holder, .. } => holder.clone(), Cmd::QuoteVerification { target, .. } => target.clone(), + Cmd::PeerConsideredAsBad { bad_peer, .. } => bad_peer.clone(), } } } @@ -85,6 +102,15 @@ impl std::fmt::Display for Cmd { quotes.len() ) } + Cmd::PeerConsideredAsBad { + detected_by, + bad_peer, + bad_behaviour, + } => { + write!( + f, + "Cmd::PeerConsideredAsBad({detected_by:?} consider peer {bad_peer:?} as bad, due to {bad_behaviour:?})") + } } } } diff --git a/sn_protocol/src/messages/response.rs b/sn_protocol/src/messages/response.rs index 46bef9937c..49635be740 100644 --- a/sn_protocol/src/messages/response.rs +++ b/sn_protocol/src/messages/response.rs @@ -111,6 +111,11 @@ pub enum CmdResponse { // /// Response to quote verification cmd QuoteVerification(Result<()>), + // + // ===== PeerConsideredAsBad ===== + // + /// Response to the considered as bad notification + PeerConsideredAsBad(Result<()>), } /// The Ok variant of a CmdResponse From 2a796eee14217c42c6f51e90281f96965be33e88 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Wed, 10 Apr 2024 13:08:14 +0900 Subject: [PATCH 038/205] feat(faucet): fully limit any concurrency This means we'll be able to await any verification steps and retries (once we have that working properly) to avoid spend errors there (as wallet lock releases quite often) --- sn_faucet/src/faucet_server.rs | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/sn_faucet/src/faucet_server.rs b/sn_faucet/src/faucet_server.rs index 05dca38b00..2818a99a3e 100644 --- a/sn_faucet/src/faucet_server.rs +++ b/sn_faucet/src/faucet_server.rs @@ -15,8 +15,9 @@ use sn_client::Client; use sn_transfers::{ get_faucet_data_dir, wallet_lockfile_name, HotWallet, NanoTokens, WALLET_DIR_NAME, }; -use std::collections::HashMap; use std::path::Path; +use std::{collections::HashMap, sync::Arc}; +use tokio::sync::Semaphore; use tracing::{debug, error, info, warn}; use warp::{ http::{Response, StatusCode}, @@ -69,9 +70,12 @@ async fn respond_to_distribution_request( client: Client, query: HashMap, balances: HashMap, + semaphore: Arc, ) -> std::result::Result { + let permit = semaphore.try_acquire(); + // some rate limiting - if is_wallet_locked() { + if is_wallet_locked() || permit.is_err() { warn!("Rate limited request due to locked wallet"); let mut response = Response::new("Rate limited".to_string()); @@ -119,10 +123,13 @@ fn is_wallet_locked() -> bool { async fn respond_to_gift_request( client: Client, key: String, + semaphore: Arc, ) -> std::result::Result { + let permit = semaphore.try_acquire(); + // some rate limiting - if is_wallet_locked() { - warn!("Rate limited request due to locked wallet"); + if is_wallet_locked() || permit.is_err() { + warn!("Rate limited request due"); let mut response = Response::new("Rate limited".to_string()); *response.status_mut() = StatusCode::TOO_MANY_REQUESTS; @@ -146,6 +153,9 @@ async fn respond_to_gift_request( } async fn startup_server(client: Client) -> Result<()> { + // Create a semaphore with a single permit + let semaphore = Arc::new(Semaphore::new(1)); + #[allow(unused)] let mut balances = HashMap::::new(); #[cfg(feature = "distribution")] @@ -163,6 +173,9 @@ async fn startup_server(client: Client) -> Result<()> { } let gift_client = client.clone(); + #[cfg(feature = "distribution")] + let semaphore_dist = semaphore.clone(); + // GET /distribution/address=address&wallet=wallet&signature=signature #[cfg(feature = "distribution")] let distribution_route = warp::get() @@ -173,8 +186,9 @@ async fn startup_server(client: Client) -> Result<()> { query }) .and_then(move |query| { + let semaphore = semaphore_dist.clone(); let client = client.clone(); - respond_to_distribution_request(client, query, balances.clone()) + respond_to_distribution_request(client, query, balances.clone(), semaphore) }); // GET /key @@ -186,8 +200,9 @@ async fn startup_server(client: Client) -> Result<()> { }) .and_then(move |key| { let client = gift_client.clone(); + let semaphore = semaphore.clone(); - respond_to_gift_request(client, key) + respond_to_gift_request(client, key, semaphore) }); println!("Starting http server listening on port 8000..."); From a60edf10afad55fb8012f9f0a3b1b541f2aee8c8 Mon Sep 17 00:00:00 2001 From: grumbach Date: Wed, 3 Apr 2024 12:01:47 +0900 Subject: [PATCH 039/205] feat: rename token to amount in Spend BREAKING CHANGE: field renamed in Spend --- sn_cli/src/bin/subcommands/wallet/hot_wallet.rs | 2 +- sn_transfers/src/cashnotes/builder.rs | 4 ++-- sn_transfers/src/cashnotes/signed_spend.rs | 8 ++++---- sn_transfers/src/genesis.rs | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs b/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs index a45749dcef..a60a51d394 100644 --- a/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs +++ b/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs @@ -293,7 +293,7 @@ fn sign_transaction(tx: &str, root_dir: &Path, force: bool) -> Result<()> { for (i, (spend, _)) in unsigned_transfer.spends.iter().enumerate() { println!("\nSpending input #{i}:"); println!("\tKey: {}", spend.unique_pubkey.to_hex()); - println!("\tAmount: {}", spend.token); + println!("\tAmount: {}", spend.amount); if let Some(ref tx) = spent_tx { if tx != &spend.spent_tx { bail!("Transaction seems corrupted, not all Spends (inputs) refer to the same transaction"); diff --git a/sn_transfers/src/cashnotes/builder.rs b/sn_transfers/src/cashnotes/builder.rs index ccfd41ce54..0f51a7b50f 100644 --- a/sn_transfers/src/cashnotes/builder.rs +++ b/sn_transfers/src/cashnotes/builder.rs @@ -119,7 +119,7 @@ impl TransactionBuilder { unique_pubkey: *input.unique_pubkey(), spent_tx: spent_tx.clone(), reason, - token: input.amount, + amount: input.amount, parent_tx: input_src_tx.clone(), network_royalties: network_royalties.clone(), }; @@ -158,7 +158,7 @@ impl TransactionBuilder { unique_pubkey: *input.unique_pubkey(), spent_tx: tx.clone(), reason, - token: input.amount, + amount: input.amount, parent_tx: input_src_tx.clone(), network_royalties: network_royalties.clone(), }; diff --git a/sn_transfers/src/cashnotes/signed_spend.rs b/sn_transfers/src/cashnotes/signed_spend.rs index 1eabc1ddaf..174060d202 100644 --- a/sn_transfers/src/cashnotes/signed_spend.rs +++ b/sn_transfers/src/cashnotes/signed_spend.rs @@ -51,7 +51,7 @@ impl SignedSpend { /// Get Nano pub fn token(&self) -> &NanoTokens { - &self.spend.token + &self.spend.amount } /// Get reason. @@ -113,7 +113,7 @@ impl SignedSpend { } // check that the value of the spend wasn't tampered with - let claimed_value = self.spend.token; + let claimed_value = self.spend.amount; let creation_value = self .spend .parent_tx @@ -214,7 +214,7 @@ pub struct Spend { pub reason: Hash, /// The amount of the input CashNote. #[debug(skip)] - pub token: NanoTokens, + pub amount: NanoTokens, /// The transaction that the input CashNote was created in (where it is an output) #[debug(skip)] pub parent_tx: Transaction, @@ -231,7 +231,7 @@ impl Spend { bytes.extend(self.unique_pubkey.to_bytes()); bytes.extend(self.spent_tx.hash().as_ref()); bytes.extend(self.reason.as_ref()); - bytes.extend(self.token.to_bytes()); + bytes.extend(self.amount.to_bytes()); bytes.extend(self.parent_tx.hash().as_ref()); bytes } diff --git a/sn_transfers/src/genesis.rs b/sn_transfers/src/genesis.rs index 61119c9af0..189ac344fc 100644 --- a/sn_transfers/src/genesis.rs +++ b/sn_transfers/src/genesis.rs @@ -92,7 +92,7 @@ pub fn is_genesis_spend(spend: &SignedSpend) -> bool { .unique_pubkey() .verify(&spend.derived_key_sig, bytes) && is_genesis_parent_tx(&spend.spend.parent_tx) - && spend.spend.token == NanoTokens::from(GENESIS_CASHNOTE_AMOUNT) + && spend.spend.amount == NanoTokens::from(GENESIS_CASHNOTE_AMOUNT) } pub fn load_genesis_wallet() -> Result { From 14c5b1b1e6cad2e3f19ddbc8e8168b59f30c4673 Mon Sep 17 00:00:00 2001 From: grumbach Date: Tue, 9 Apr 2024 15:37:32 +0900 Subject: [PATCH 040/205] feat: renamings in CashNote BREAKING CHANGE: rename fields in CashNote --- sn_cli/src/bin/subcommands/wallet.rs | 2 +- sn_client/src/audit/spend_check.rs | 8 ++--- sn_client/src/audit/spend_dag_building.rs | 8 ++--- sn_client/src/wallet.rs | 4 +-- sn_networking/src/transfers.rs | 6 ++-- sn_transfers/src/cashnotes.rs | 24 ++++++------- sn_transfers/src/cashnotes/builder.rs | 6 ++-- sn_transfers/src/cashnotes/cashnote.rs | 35 ++++++++++--------- sn_transfers/src/genesis.rs | 2 +- .../src/transfers/offline_transfer.rs | 4 +-- sn_transfers/src/transfers/transfer.rs | 2 +- sn_transfers/src/wallet/watch_only.rs | 7 +++- 12 files changed, 57 insertions(+), 51 deletions(-) diff --git a/sn_cli/src/bin/subcommands/wallet.rs b/sn_cli/src/bin/subcommands/wallet.rs index b8a88f036a..a4382b9bb4 100644 --- a/sn_cli/src/bin/subcommands/wallet.rs +++ b/sn_cli/src/bin/subcommands/wallet.rs @@ -61,7 +61,7 @@ impl WalletApiHelper { let cash_notes = vec![cash_note.clone()]; let spent_unique_pubkeys: BTreeSet<_> = cash_note - .src_tx + .parent_tx .inputs .iter() .map(|input| input.unique_pubkey()) diff --git a/sn_client/src/audit/spend_check.rs b/sn_client/src/audit/spend_check.rs index e2df4a4a7b..a8443b3d89 100644 --- a/sn_client/src/audit/spend_check.rs +++ b/sn_client/src/audit/spend_check.rs @@ -86,10 +86,10 @@ impl Client { trace!("Spends for {parent_tx_hash:?} - {spends:?}"); // check if we reached the genesis Tx - if parent_tx == sn_transfers::GENESIS_CASHNOTE.src_tx - && spends - .iter() - .all(|s| s.spend.unique_pubkey == sn_transfers::GENESIS_CASHNOTE.id) + if parent_tx == sn_transfers::GENESIS_CASHNOTE.parent_tx + && spends.iter().all(|s| { + s.spend.unique_pubkey == sn_transfers::GENESIS_CASHNOTE.unique_pubkey + }) && spends.len() == 1 { debug!("Depth {depth} - Reached genesis Tx on one branch: {parent_tx_hash:?}"); diff --git a/sn_client/src/audit/spend_dag_building.rs b/sn_client/src/audit/spend_dag_building.rs index 6fb7737523..d10417cbe4 100644 --- a/sn_client/src/audit/spend_dag_building.rs +++ b/sn_client/src/audit/spend_dag_building.rs @@ -194,10 +194,10 @@ impl Client { trace!("Spends for {parent_tx_hash:?} - {spends:?}"); // check if we reached the genesis Tx - if parent_tx == sn_transfers::GENESIS_CASHNOTE.src_tx - && spends - .iter() - .all(|s| s.spend.unique_pubkey == sn_transfers::GENESIS_CASHNOTE.id) + if parent_tx == sn_transfers::GENESIS_CASHNOTE.parent_tx + && spends.iter().all(|s| { + s.spend.unique_pubkey == sn_transfers::GENESIS_CASHNOTE.unique_pubkey + }) && spends.len() == 1 { debug!("Depth {depth} - reached genesis Tx on one branch: {parent_tx_hash:?}"); diff --git a/sn_client/src/wallet.rs b/sn_client/src/wallet.rs index 3ece569a77..eb9c9f7108 100644 --- a/sn_client/src/wallet.rs +++ b/sn_client/src/wallet.rs @@ -973,7 +973,7 @@ impl Client { // and compare them to the spends in the cash_note, to know if the // transfer is considered valid in the network. let mut tasks = Vec::new(); - for spend in &cash_note.signed_spends { + for spend in &cash_note.parent_spends { let address = SpendAddress::from_unique_pubkey(spend.unique_pubkey()); debug!( "Getting spend for pubkey {:?} from network at {address:?}", @@ -991,7 +991,7 @@ impl Client { // If all the spends in the cash_note are the same as the ones in the network, // we have successfully verified that the cash_note is globally recognised and therefor valid. - if received_spends == cash_note.signed_spends { + if received_spends == cash_note.parent_spends { return Ok(()); } Err(WalletError::CouldNotVerifyTransfer( diff --git a/sn_networking/src/transfers.rs b/sn_networking/src/transfers.rs index 0d1d08a601..15e739a595 100644 --- a/sn_networking/src/transfers.rs +++ b/sn_networking/src/transfers.rs @@ -166,9 +166,9 @@ impl Network { .cloned() .collect(); let cash_note = CashNote { - id, - src_tx, - signed_spends, + unique_pubkey: id, + parent_tx: src_tx, + parent_spends: signed_spends, main_pubkey, derivation_index, }; diff --git a/sn_transfers/src/cashnotes.rs b/sn_transfers/src/cashnotes.rs index b865580af2..9920a0f2bd 100644 --- a/sn_transfers/src/cashnotes.rs +++ b/sn_transfers/src/cashnotes.rs @@ -46,9 +46,9 @@ pub(crate) mod tests { outputs: vec![Output::new(derived_key.unique_pubkey(), amount)], }; let cashnote = CashNote { - id: derived_key.unique_pubkey(), - src_tx: tx, - signed_spends: Default::default(), + unique_pubkey: derived_key.unique_pubkey(), + parent_tx: tx, + parent_spends: Default::default(), main_pubkey: main_key.main_pubkey(), derivation_index, }; @@ -73,9 +73,9 @@ pub(crate) mod tests { outputs: vec![Output::new(derived_key.unique_pubkey(), amount)], }; let cashnote = CashNote { - id: derived_key.unique_pubkey(), - src_tx: tx, - signed_spends: Default::default(), + unique_pubkey: derived_key.unique_pubkey(), + parent_tx: tx, + parent_spends: Default::default(), main_pubkey: main_key.main_pubkey(), derivation_index, }; @@ -104,9 +104,9 @@ pub(crate) mod tests { }; let cashnote = CashNote { - id: derived_key.unique_pubkey(), - src_tx: tx, - signed_spends: Default::default(), + unique_pubkey: derived_key.unique_pubkey(), + parent_tx: tx, + parent_spends: Default::default(), main_pubkey: main_key.main_pubkey(), derivation_index, }; @@ -135,9 +135,9 @@ pub(crate) mod tests { }; let cashnote = CashNote { - id: derived_key.unique_pubkey(), - src_tx: tx, - signed_spends: Default::default(), + unique_pubkey: derived_key.unique_pubkey(), + parent_tx: tx, + parent_spends: Default::default(), main_pubkey: main_key.main_pubkey(), derivation_index, }; diff --git a/sn_transfers/src/cashnotes/builder.rs b/sn_transfers/src/cashnotes/builder.rs index 0f51a7b50f..d43d86d692 100644 --- a/sn_transfers/src/cashnotes/builder.rs +++ b/sn_transfers/src/cashnotes/builder.rs @@ -235,9 +235,9 @@ impl CashNoteBuilder { Ok(( CashNote { - id: main_pubkey.new_unique_pubkey(derivation_index), - src_tx: self.spent_tx.clone(), - signed_spends: self.signed_spends.clone(), + unique_pubkey: main_pubkey.new_unique_pubkey(derivation_index), + parent_tx: self.spent_tx.clone(), + parent_spends: self.signed_spends.clone(), main_pubkey: *main_pubkey, derivation_index: *derivation_index, }, diff --git a/sn_transfers/src/cashnotes/cashnote.rs b/sn_transfers/src/cashnotes/cashnote.rs index 7ba02b38c4..dee98d65ca 100644 --- a/sn_transfers/src/cashnotes/cashnote.rs +++ b/sn_transfers/src/cashnotes/cashnote.rs @@ -58,25 +58,26 @@ use tiny_keccak::{Hasher, Sha3}; /// eg: `cashnote.derivation_index(&main_key)` #[derive(custom_debug::Debug, Clone, Eq, PartialEq, Serialize, Deserialize, Hash)] pub struct CashNote { - /// The unique pulbic key of this CashNote. It is unique, and there can never + /// The unique public key of this CashNote. It is unique, and there can never /// be another CashNote with the same pulbic key. It used in SignedSpends. - pub id: UniquePubkey, + pub unique_pubkey: UniquePubkey, /// The transaction where this CashNote was created. #[debug(skip)] - pub src_tx: Transaction, + pub parent_tx: Transaction, /// The transaction's input's SignedSpends - pub signed_spends: BTreeSet, - /// This is the MainPubkey of the recipient of this CashNote. + pub parent_spends: BTreeSet, + /// This is the MainPubkey of the owner of this CashNote pub main_pubkey: MainPubkey, - /// This indicates which index to use when deriving the UniquePubkey of the - /// CashNote, from the MainPubkey. + /// The derivation index used to derive the UniquePubkey and DerivedSecretKey from the MainPubkey and MainSecretKey respectively. + /// It is to be kept secret to preserve the privacy of the owner. + /// Without it, it is very hard to link the MainPubkey (original owner) and the UniquePubkey (derived unique identity of the CashNote) pub derivation_index: DerivationIndex, } impl CashNote { - /// Return the id of this CashNote. + /// Return the unique pubkey of this CashNote. pub fn unique_pubkey(&self) -> UniquePubkey { - self.id + self.unique_pubkey } // Return MainPubkey from which UniquePubkey is derived. @@ -112,7 +113,7 @@ impl CashNote { /// Return the reason why this CashNote was spent. /// Will be the default Hash (empty) if reason is none. pub fn reason(&self) -> Hash { - self.signed_spends + self.parent_spends .iter() .next() .map(|c| c.reason()) @@ -122,7 +123,7 @@ impl CashNote { /// Return the value in NanoTokens for this CashNote. pub fn value(&self) -> Result { Ok(self - .src_tx + .parent_tx .outputs .iter() .find(|o| &self.unique_pubkey() == o.unique_pubkey()) @@ -133,11 +134,11 @@ impl CashNote { /// Generate the hash of this CashNote pub fn hash(&self) -> Hash { let mut sha3 = Sha3::v256(); - sha3.update(self.src_tx.hash().as_ref()); + sha3.update(self.parent_tx.hash().as_ref()); sha3.update(&self.main_pubkey.to_bytes()); sha3.update(&self.derivation_index.0); - for sp in self.signed_spends.iter() { + for sp in self.parent_spends.iter() { sha3.update(&sp.to_bytes()); } @@ -161,12 +162,12 @@ impl CashNote { /// see TransactionVerifier::verify() for a description of /// verifier requirements. pub fn verify(&self, main_key: &MainSecretKey) -> Result<(), TransferError> { - self.src_tx - .verify_against_inputs_spent(self.signed_spends.iter())?; + self.parent_tx + .verify_against_inputs_spent(self.parent_spends.iter())?; let unique_pubkey = self.derived_key(main_key)?.unique_pubkey(); if !self - .src_tx + .parent_tx .outputs .iter() .any(|o| unique_pubkey.eq(o.unique_pubkey())) @@ -177,7 +178,7 @@ impl CashNote { // verify that all signed_spend reasons are equal let reason = self.reason(); let reasons_are_equal = |s: &SignedSpend| reason == s.reason(); - if !self.signed_spends.iter().all(reasons_are_equal) { + if !self.parent_spends.iter().all(reasons_are_equal) { return Err(TransferError::SignedSpendReasonMismatch(unique_pubkey)); } Ok(()) diff --git a/sn_transfers/src/genesis.rs b/sn_transfers/src/genesis.rs index 189ac344fc..6b36a03bdc 100644 --- a/sn_transfers/src/genesis.rs +++ b/sn_transfers/src/genesis.rs @@ -81,7 +81,7 @@ lazy_static! { /// Return if provided Transaction is genesis parent tx. pub fn is_genesis_parent_tx(parent_tx: &Transaction) -> bool { - parent_tx == &GENESIS_CASHNOTE.src_tx + parent_tx == &GENESIS_CASHNOTE.parent_tx } /// Return if provided Spend is genesis spend. diff --git a/sn_transfers/src/transfers/offline_transfer.rs b/sn_transfers/src/transfers/offline_transfer.rs index ca44440740..5a43a1fc96 100644 --- a/sn_transfers/src/transfers/offline_transfer.rs +++ b/sn_transfers/src/transfers/offline_transfer.rs @@ -258,10 +258,10 @@ fn create_transaction_builder_with( inputs.push(( input, derived_key, - cash_note.src_tx.clone(), + cash_note.parent_tx.clone(), cash_note.derivation_index, )); - let _ = src_txs.insert(cash_note.unique_pubkey(), cash_note.src_tx); + let _ = src_txs.insert(cash_note.unique_pubkey(), cash_note.parent_tx); } // Build the transaction and create change cash_note if needed diff --git a/sn_transfers/src/transfers/transfer.rs b/sn_transfers/src/transfers/transfer.rs index 2a76692232..89f06c0633 100644 --- a/sn_transfers/src/transfers/transfer.rs +++ b/sn_transfers/src/transfers/transfer.rs @@ -151,7 +151,7 @@ impl CashNoteRedemption { pub fn from_cash_note(cash_note: &CashNote) -> Result { let derivation_index = cash_note.derivation_index(); - let parent_spend = match cash_note.signed_spends.iter().next() { + let parent_spend = match cash_note.parent_spends.iter().next() { Some(s) => SpendAddress::from_unique_pubkey(s.unique_pubkey()), None => { return Err(TransferError::CashNoteHasNoParentSpends); diff --git a/sn_transfers/src/wallet/watch_only.rs b/sn_transfers/src/wallet/watch_only.rs index 38d4338611..8cfdc58923 100644 --- a/sn_transfers/src/wallet/watch_only.rs +++ b/sn_transfers/src/wallet/watch_only.rs @@ -74,7 +74,12 @@ impl WatchOnlyWallet { let cash_notes = load_cash_notes_from_disk(&self.wallet_dir)?; let spent_unique_pubkeys: BTreeSet<_> = cash_notes .iter() - .flat_map(|cn| cn.src_tx.inputs.iter().map(|input| input.unique_pubkey())) + .flat_map(|cn| { + cn.parent_tx + .inputs + .iter() + .map(|input| input.unique_pubkey()) + }) .collect(); self.deposit(&cash_notes)?; self.mark_notes_as_spent(spent_unique_pubkeys); From 0e92c1d4c217200a1d052f2f5d38615865a2cbe7 Mon Sep 17 00:00:00 2001 From: grumbach Date: Tue, 9 Apr 2024 15:40:05 +0900 Subject: [PATCH 041/205] fix: typo --- sn_transfers/src/cashnotes/cashnote.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sn_transfers/src/cashnotes/cashnote.rs b/sn_transfers/src/cashnotes/cashnote.rs index dee98d65ca..d14a5886de 100644 --- a/sn_transfers/src/cashnotes/cashnote.rs +++ b/sn_transfers/src/cashnotes/cashnote.rs @@ -59,7 +59,7 @@ use tiny_keccak::{Hasher, Sha3}; #[derive(custom_debug::Debug, Clone, Eq, PartialEq, Serialize, Deserialize, Hash)] pub struct CashNote { /// The unique public key of this CashNote. It is unique, and there can never - /// be another CashNote with the same pulbic key. It used in SignedSpends. + /// be another CashNote with the same public key. It used in SignedSpends. pub unique_pubkey: UniquePubkey, /// The transaction where this CashNote was created. #[debug(skip)] From 4658048772a1fac699af7a15f9a0dcfebf12ac8f Mon Sep 17 00:00:00 2001 From: qima Date: Wed, 10 Apr 2024 22:46:02 +0800 Subject: [PATCH 042/205] test(node): unit test for recover historic quoting metrics --- sn_networking/src/record_store.rs | 40 +++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/sn_networking/src/record_store.rs b/sn_networking/src/record_store.rs index d5cb5e8b15..24004ba5ca 100644 --- a/sn_networking/src/record_store.rs +++ b/sn_networking/src/record_store.rs @@ -1230,6 +1230,46 @@ mod tests { Ok(()) } + #[tokio::test] + async fn historic_quoting_metrics() -> Result<()> { + let temp_dir = std::env::temp_dir(); + let unique_dir_name = uuid::Uuid::new_v4().to_string(); + let storage_dir = temp_dir.join(unique_dir_name); + fs::create_dir_all(&storage_dir).expect("Failed to create directory"); + + let store_config = NodeRecordStoreConfig { + storage_dir, + ..Default::default() + }; + let self_id = PeerId::random(); + let (network_event_sender, _) = mpsc::channel(1); + let (swarm_cmd_sender, _) = mpsc::channel(1); + + let mut store = NodeRecordStore::with_config( + self_id, + store_config.clone(), + network_event_sender.clone(), + swarm_cmd_sender.clone(), + ); + + store.payment_received(); + + // Wait for a while to allow the file written to disk. + sleep(Duration::from_millis(5000)).await; + + let new_store = NodeRecordStore::with_config( + self_id, + store_config, + network_event_sender, + swarm_cmd_sender, + ); + + assert_eq!(1, new_store.received_payment_count); + assert_eq!(store.timestamp, new_store.timestamp); + + Ok(()) + } + #[test] fn address_distribution_sim() { // Map of peers and correspondent stats of `(num_of_records, Nano_earned, received_payment_count)`. From 5c670793496e5e6f2fbe058cba0929c8984b2663 Mon Sep 17 00:00:00 2001 From: qima Date: Wed, 10 Apr 2024 19:11:03 +0800 Subject: [PATCH 043/205] fix(node): not send out replication when failed read from local --- sn_networking/src/cmd.rs | 5 +++- sn_networking/src/replication_fetcher.rs | 29 ++++++++++++++++-------- sn_node/src/replication.rs | 4 ++-- sn_node/tests/verify_data_location.rs | 10 +++++--- 4 files changed, 33 insertions(+), 15 deletions(-) diff --git a/sn_networking/src/cmd.rs b/sn_networking/src/cmd.rs index 87c18dd2c2..744c43f2bc 100644 --- a/sn_networking/src/cmd.rs +++ b/sn_networking/src/cmd.rs @@ -493,7 +493,10 @@ impl SwarmDriver { }; } SwarmCmd::AddLocalRecordAsStored { key, record_type } => { - trace!("Adding Record locally, for {key:?} and {record_type:?}"); + info!( + "Adding Record locally, for {:?} and {record_type:?}", + PrettyPrintRecordKey::from(&key) + ); cmd_string = "AddLocalRecordAsStored"; self.swarm .behaviour_mut() diff --git a/sn_networking/src/replication_fetcher.rs b/sn_networking/src/replication_fetcher.rs index 046b3d693c..a5cdbdafc5 100644 --- a/sn_networking/src/replication_fetcher.rs +++ b/sn_networking/src/replication_fetcher.rs @@ -231,16 +231,27 @@ impl ReplicationFetcher { // 1, the pending_entries from that node shall be removed from `to_be_fetched` list. // 2, firing event up to notify bad_nodes, hence trigger them to be removed from RT. fn prune_expired_keys_and_slow_nodes(&mut self) { - let mut failed_holders = BTreeSet::default(); + let mut failed_fetches = vec![]; + + self.on_going_fetches + .retain(|(record_key, _), (peer_id, time_out)| { + if *time_out < Instant::now() { + failed_fetches.push((record_key.clone(), *peer_id)); + false + } else { + true + } + }); - self.on_going_fetches.retain(|_, (peer_id, time_out)| { - if *time_out < Instant::now() { - failed_holders.insert(*peer_id); - false - } else { - true - } - }); + let mut failed_holders = BTreeSet::new(); + + for (record_key, peer_id) in failed_fetches { + error!( + "Failed to fetch {:?} from {peer_id:?}", + PrettyPrintRecordKey::from(&record_key) + ); + let _ = failed_holders.insert(peer_id); + } // now to clear any failed nodes from our lists. self.to_be_fetched diff --git a/sn_node/src/replication.rs b/sn_node/src/replication.rs index 4de65952ff..d5e0c1bee9 100644 --- a/sn_node/src/replication.rs +++ b/sn_node/src/replication.rs @@ -112,7 +112,7 @@ impl Node { error!( "Replicating fresh record {pretty_key:?} get_record_from_store errored: {err:?}" ); - return; + None } }; @@ -124,7 +124,7 @@ impl Node { error!( "Could not get record from store for replication: {pretty_key:?} after 10 retries" ); - break; + return; } retry_count += 1; diff --git a/sn_node/tests/verify_data_location.rs b/sn_node/tests/verify_data_location.rs index e8bed364b6..64edc91138 100644 --- a/sn_node/tests/verify_data_location.rs +++ b/sn_node/tests/verify_data_location.rs @@ -270,7 +270,7 @@ async fn verify_location(all_peers: &Vec, node_rpc_addresses: &[SocketAd .for_each(|expected| failed_peers.push(*expected)); if !failed_peers.is_empty() { - failed.insert(PrettyPrintRecordKey::from(key).into_owned(), failed_peers); + failed.insert(key.clone(), failed_peers); } } @@ -279,9 +279,13 @@ async fn verify_location(all_peers: &Vec, node_rpc_addresses: &[SocketAd println!("Verification failed for {:?} entries", failed.len()); failed.iter().for_each(|(key, failed_peers)| { + let key_addr = NetworkAddress::from_record_key(key); + let pretty_key = PrettyPrintRecordKey::from(key); failed_peers.iter().for_each(|peer| { - println!("Record {key:?} is not stored inside {peer:?}"); - error!("Record {key:?} is not stored inside {peer:?}"); + let peer_addr = NetworkAddress::from_peer(*peer); + let ilog2_distance = peer_addr.distance(&key_addr).ilog2(); + println!("Record {pretty_key:?} is not stored inside {peer:?}, with ilog2 distance to be {ilog2_distance:?}"); + error!("Record {pretty_key:?} is not stored inside {peer:?}, with ilog2 distance to be {ilog2_distance:?}"); }); }); info!("State of each node:"); From c96cc5b174c6ef88f6593bb480c33e4f6fddf007 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 9 Apr 2024 15:53:23 +0900 Subject: [PATCH 044/205] feat(resources): update bump script to strip suffix-version if no changes found --- resources/scripts/bump_version.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/resources/scripts/bump_version.sh b/resources/scripts/bump_version.sh index 748f2f3019..bdec0d4999 100755 --- a/resources/scripts/bump_version.sh +++ b/resources/scripts/bump_version.sh @@ -31,8 +31,19 @@ done < <(cat bump_version_output | grep "^\*") len=${#crates_bumped[@]} if [[ $len -eq 0 ]]; then - echo "No changes detected. Exiting without bumping any versions." - exit 0 + echo "No changes detected." + if [[ -z "$SUFFIX" ]]; then + echo "Removing any existing suffixes and bumping versions to stable." + for crate in $(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | .name'); do + version=$(cargo metadata --no-deps --format-version 1 | jq -r --arg crate_name "$crate" '.packages[] | select(.name==$crate_name) | .version') + new_version=$(echo "$version" | sed -E 's/(-alpha\.[0-9]+|-beta\.[0-9]+)$//') + if [[ "$version" != "$new_version" ]]; then + echo "Removing suffix from $crate, setting version to $new_version" + cargo set-version -p $crate $new_version + crates_bumped+=("${crate}-v${new_version}") + fi + done + fi fi if [[ -n "$SUFFIX" ]]; then From 9d34b01607c10ef06a3406cd7ee7164f81873627 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 11 Apr 2024 13:00:07 +0900 Subject: [PATCH 045/205] ci: add workflow dispatch for version bumper, with branch option --- .github/workflows/version_bump.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml index ff3e5a8548..464410e643 100644 --- a/.github/workflows/version_bump.yml +++ b/.github/workflows/version_bump.yml @@ -1,5 +1,5 @@ -# Automatically run the versino bump script. -# We do this only on main/beta branches, and only if the commit message doesn't start with 'chore(release):' +# Automatically run the version bump script. +# We do this only on main/beta branches, and also allow manual triggers on specific branches via workflow dispatch. name: Version Bump # prevent concurrent version bumps @@ -10,6 +10,12 @@ on: schedule: # Schedule for midnight UTC. Adjust according to your timezone - cron: "0 0 * * *" # Runs at 00:00 UTC every day + workflow_dispatch: + inputs: + branch: + description: 'Branch to run version bump on' + required: true + default: 'alpha' env: RELEASE_PLZ_BIN_URL: https://github.com/MarcoIeni/release-plz/releases/download/release-plz-v0.3.43/release-plz-x86_64-unknown-linux-gnu.tar.gz @@ -27,8 +33,8 @@ jobs: with: fetch-depth: "0" token: ${{ secrets.VERSION_BUMP_COMMIT_PAT }} - # For scheduled runs, explicitly set ref to 'main'. Otherwise, use the triggering ref. - ref: ${{ github.event_name == 'schedule' && 'main' || github.ref_name }} + # For scheduled runs, explicitly set ref to 'main'. For workflow_dispatch, use the input branch. Otherwise, use the triggering ref. + ref: ${{ github.event_name == 'schedule' && 'main' || github.event.inputs.branch || github.ref_name }} - name: Get the SHA of the last release commit id: get-sha @@ -45,11 +51,11 @@ jobs: repository-name: ${{ github.repository }} repository-owner: ${{ github.repository_owner }} head-sha: ${{ env.sha }} - base-branch: ${{ github.event_name == 'schedule' && 'main' || github.ref_name }} + base-branch: ${{ github.event_name == 'schedule' && 'main' || github.event.inputs.branch || github.ref_name }} polling-interval: 60 - name: Fetch the latest code from the specified branch - run: git pull origin ${{ github.event_name == 'schedule' && 'main' || github.ref_name }} + run: git pull origin ${{ github.event_name == 'schedule' && 'main' || github.event.inputs.branch || github.ref_name }} - uses: actions-rs/toolchain@v1 with: @@ -76,12 +82,12 @@ jobs: sudo mv release-plz /usr/local/bin - shell: bash # run an alpha release bump when scheduled, otherwise run as the branch name - run: ./resources/scripts/bump_version.sh ${{ github.event_name == 'schedule' && 'alpha' || github.ref_name }} + run: ./resources/scripts/bump_version.sh ${{ github.event_name == 'schedule' && 'alpha' || github.event.inputs.branch || github.ref_name }} - name: push version bump commit uses: ad-m/github-push-action@master with: github_token: ${{ secrets.VERSION_BUMP_COMMIT_PAT }} - branch: ${{ github.event_name == 'schedule' && 'main' || github.ref_name }} + branch: ${{ github.event_name == 'schedule' && 'main' || github.event.inputs.branch || github.ref_name }} tags: true - name: post notification to slack on failure if: ${{ failure() }} From 37ab87b2a410bab7b45e8a5562bc7c7605f7e5e6 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 11 Apr 2024 13:01:40 +0900 Subject: [PATCH 046/205] ci: remove schedule version bumper from workflow --- .github/workflows/version_bump.yml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml index 464410e643..00ad344238 100644 --- a/.github/workflows/version_bump.yml +++ b/.github/workflows/version_bump.yml @@ -4,12 +4,9 @@ name: Version Bump # prevent concurrent version bumps concurrency: - group: "version-bump-${{ github.event_name == 'schedule' && 'main' || github.ref_name }}" + group: "version-bump-${{ github.ref_name }}" on: - schedule: - # Schedule for midnight UTC. Adjust according to your timezone - - cron: "0 0 * * *" # Runs at 00:00 UTC every day workflow_dispatch: inputs: branch: @@ -33,8 +30,8 @@ jobs: with: fetch-depth: "0" token: ${{ secrets.VERSION_BUMP_COMMIT_PAT }} - # For scheduled runs, explicitly set ref to 'main'. For workflow_dispatch, use the input branch. Otherwise, use the triggering ref. - ref: ${{ github.event_name == 'schedule' && 'main' || github.event.inputs.branch || github.ref_name }} + # For workflow_dispatch, use the input branch. Otherwise, use the triggering ref. + ref: ${{ github.event.inputs.branch || github.ref_name }} - name: Get the SHA of the last release commit id: get-sha @@ -51,11 +48,11 @@ jobs: repository-name: ${{ github.repository }} repository-owner: ${{ github.repository_owner }} head-sha: ${{ env.sha }} - base-branch: ${{ github.event_name == 'schedule' && 'main' || github.event.inputs.branch || github.ref_name }} + base-branch: ${{ github.event.inputs.branch || github.ref_name }} polling-interval: 60 - name: Fetch the latest code from the specified branch - run: git pull origin ${{ github.event_name == 'schedule' && 'main' || github.event.inputs.branch || github.ref_name }} + run: git pull origin ${{ github.event.inputs.branch || github.ref_name }} - uses: actions-rs/toolchain@v1 with: @@ -81,13 +78,13 @@ jobs: rm release-plz-x86_64-unknown-linux-gnu.tar.gz sudo mv release-plz /usr/local/bin - shell: bash - # run an alpha release bump when scheduled, otherwise run as the branch name - run: ./resources/scripts/bump_version.sh ${{ github.event_name == 'schedule' && 'alpha' || github.event.inputs.branch || github.ref_name }} + # run as the branch name + run: ./resources/scripts/bump_version.sh ${{ github.event.inputs.branch || github.ref_name }} - name: push version bump commit uses: ad-m/github-push-action@master with: github_token: ${{ secrets.VERSION_BUMP_COMMIT_PAT }} - branch: ${{ github.event_name == 'schedule' && 'main' || github.event.inputs.branch || github.ref_name }} + branch: ${{ github.event.inputs.branch || github.ref_name }} tags: true - name: post notification to slack on failure if: ${{ failure() }} From 450cc4afc6950e4ef2a71a03ea11a3c5a524e258 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 11 Apr 2024 14:15:02 +0900 Subject: [PATCH 047/205] ci: simplify version bump dispatch --- .github/workflows/version_bump.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml index 00ad344238..9eb7d8f622 100644 --- a/.github/workflows/version_bump.yml +++ b/.github/workflows/version_bump.yml @@ -7,12 +7,7 @@ concurrency: group: "version-bump-${{ github.ref_name }}" on: - workflow_dispatch: - inputs: - branch: - description: 'Branch to run version bump on' - required: true - default: 'alpha' + workflow_dispatch: {} env: RELEASE_PLZ_BIN_URL: https://github.com/MarcoIeni/release-plz/releases/download/release-plz-v0.3.43/release-plz-x86_64-unknown-linux-gnu.tar.gz @@ -30,8 +25,7 @@ jobs: with: fetch-depth: "0" token: ${{ secrets.VERSION_BUMP_COMMIT_PAT }} - # For workflow_dispatch, use the input branch. Otherwise, use the triggering ref. - ref: ${{ github.event.inputs.branch || github.ref_name }} + ref: ${{ github.ref_name }} - name: Get the SHA of the last release commit id: get-sha @@ -48,11 +42,11 @@ jobs: repository-name: ${{ github.repository }} repository-owner: ${{ github.repository_owner }} head-sha: ${{ env.sha }} - base-branch: ${{ github.event.inputs.branch || github.ref_name }} + base-branch: ${{ github.ref_name }} polling-interval: 60 - name: Fetch the latest code from the specified branch - run: git pull origin ${{ github.event.inputs.branch || github.ref_name }} + run: git pull origin ${{ github.ref_name }} - uses: actions-rs/toolchain@v1 with: @@ -79,12 +73,12 @@ jobs: sudo mv release-plz /usr/local/bin - shell: bash # run as the branch name - run: ./resources/scripts/bump_version.sh ${{ github.event.inputs.branch || github.ref_name }} + run: ./resources/scripts/bump_version.sh ${{ github.ref_name }} - name: push version bump commit uses: ad-m/github-push-action@master with: github_token: ${{ secrets.VERSION_BUMP_COMMIT_PAT }} - branch: ${{ github.event.inputs.branch || github.ref_name }} + branch: ${{ github.ref_name }} tags: true - name: post notification to slack on failure if: ${{ failure() }} From 9ad4b38116981e9c9410f0fc23dcc163fd29ee58 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 11 Apr 2024 14:19:05 +0900 Subject: [PATCH 048/205] ci: dont require await for past release in bumper --- .github/workflows/version_bump.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml index 9eb7d8f622..2962ba8c2b 100644 --- a/.github/workflows/version_bump.yml +++ b/.github/workflows/version_bump.yml @@ -31,20 +31,6 @@ jobs: id: get-sha run: echo "sha=$(git log --grep='chore(release):' -n 1 --pretty=format:"%H")" >> $GITHUB_ENV - - name: Wait for release workflow to complete - uses: mostafahussein/workflow-watcher@v1.0.0 - # Don't fail the whole run if this step fails - # this action will fail if the previous workflows failed or was skipped, - # which isn't helpful - continue-on-error: true - with: - secret: ${{ secrets.GITHUB_TOKEN }} - repository-name: ${{ github.repository }} - repository-owner: ${{ github.repository_owner }} - head-sha: ${{ env.sha }} - base-branch: ${{ github.ref_name }} - polling-interval: 60 - - name: Fetch the latest code from the specified branch run: git pull origin ${{ github.ref_name }} From 39feb2eec29b46233b63a86f81c1e1220c750b83 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 11 Apr 2024 14:26:19 +0900 Subject: [PATCH 049/205] ci: specify suffix for release bumps --- .github/workflows/version_bump.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml index 2962ba8c2b..185a91834e 100644 --- a/.github/workflows/version_bump.yml +++ b/.github/workflows/version_bump.yml @@ -1,5 +1,4 @@ # Automatically run the version bump script. -# We do this only on main/beta branches, and also allow manual triggers on specific branches via workflow dispatch. name: Version Bump # prevent concurrent version bumps @@ -7,7 +6,12 @@ concurrency: group: "version-bump-${{ github.ref_name }}" on: - workflow_dispatch: {} + workflow_dispatch: + inputs: + suffix: + description: 'Suffix to append to the version (alpha/beta), leave empty for no suffix.' + required: false + default: 'alpha' env: RELEASE_PLZ_BIN_URL: https://github.com/MarcoIeni/release-plz/releases/download/release-plz-v0.3.43/release-plz-x86_64-unknown-linux-gnu.tar.gz @@ -58,8 +62,8 @@ jobs: rm release-plz-x86_64-unknown-linux-gnu.tar.gz sudo mv release-plz /usr/local/bin - shell: bash - # run as the branch name - run: ./resources/scripts/bump_version.sh ${{ github.ref_name }} + # run as the branch name with the suffix from workflow dispatch, allowing for an empty suffix as a valid option + run: ./resources/scripts/bump_version.sh ${{ github.event.inputs.suffix }} - name: push version bump commit uses: ad-m/github-push-action@master with: From f5e2e663210a33891b15c660850ac0536536fde4 Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 11 Apr 2024 17:56:15 +0900 Subject: [PATCH 050/205] feat: dag faults unit tests, sn_auditor offline mode --- sn_auditor/src/dag_db.rs | 32 ++-- sn_auditor/src/main.rs | 12 ++ sn_client/src/audit/spend_dag.rs | 54 +++--- sn_client/src/audit/tests/mod.rs | 287 +++++++++++++++++++++++++++++++ 4 files changed, 353 insertions(+), 32 deletions(-) diff --git a/sn_auditor/src/dag_db.rs b/sn_auditor/src/dag_db.rs index 94759c7497..081b20cbf7 100644 --- a/sn_auditor/src/dag_db.rs +++ b/sn_auditor/src/dag_db.rs @@ -26,7 +26,7 @@ pub const SPEND_DAG_SVG_FILENAME: &str = "spend_dag.svg"; /// Currently in memory, with disk backup, but should probably be a real DB at scale #[derive(Clone)] pub struct SpendDagDb { - client: Client, + client: Option, path: PathBuf, dag: Arc>, } @@ -57,7 +57,21 @@ impl SpendDagDb { }; Ok(Self { - client, + client: Some(client), + path, + dag: Arc::new(RwLock::new(dag)), + }) + } + + /// Create a new SpendDagDb from a local file and no network connection + pub fn offline(dag_path: PathBuf) -> Result { + let path = dag_path + .parent() + .ok_or_else(|| eyre!("Failed to get parent path"))? + .to_path_buf(); + let dag = SpendDag::load_from_file(&dag_path)?; + Ok(Self { + client: None, path, dag: Arc::new(RwLock::new(dag)), }) @@ -82,7 +96,7 @@ impl SpendDagDb { let (spend_type, spends) = match spend { SpendDagGet::SpendNotFound => ("SpendNotFound", vec![]), - SpendDagGet::SpendKeyExists => ("SpendKeyExists", vec![]), + SpendDagGet::Utxo => ("Utxo", vec![]), SpendDagGet::DoubleSpend(vs) => ("DoubleSpend", vs), SpendDagGet::Spend(s) => ("Spend", vec![*s]), }; @@ -146,6 +160,8 @@ impl SpendDagDb { // update that copy 10 generations further const NEXT_10_GEN: u32 = 10; self.client + .clone() + .ok_or(eyre!("Cannot update in offline mode"))? .spend_dag_continue_from_utxos(&mut dag, Some(NEXT_10_GEN)) .await?; @@ -224,17 +240,13 @@ fn quick_edit_svg(svg: Vec, dag: &SpendDag) -> Result> { let mut str = String::from_utf8(svg).map_err(|err| eyre!("Failed svg conversion: {err}"))?; let spend_addrs: Vec<_> = dag.all_spends().iter().map(|s| s.address()).collect(); - let utxo_addrs = dag.get_utxos(); - let unknown_parents = dag.get_unknown_parents(); - let all_addrs = spend_addrs - .iter() - .chain(utxo_addrs.iter()) - .chain(unknown_parents.iter()); + let pending_addrs = dag.get_pending_spends(); + let all_addrs = spend_addrs.iter().chain(pending_addrs.iter()); for addr in all_addrs { let addr_hex = addr.to_hex().to_string(); let is_fault = !dag.get_spend_faults(addr).is_empty(); - let is_known_but_not_gathered = matches!(dag.get_spend(addr), SpendDagGet::SpendKeyExists); + let is_known_but_not_gathered = matches!(dag.get_spend(addr), SpendDagGet::Utxo); let colour = if is_fault { "red" } else if is_known_but_not_gathered { diff --git a/sn_auditor/src/main.rs b/sn_auditor/src/main.rs index b9cda187b8..3c737a2345 100644 --- a/sn_auditor/src/main.rs +++ b/sn_auditor/src/main.rs @@ -21,6 +21,7 @@ use sn_client::Client; use sn_logging::{Level, LogBuilder, LogFormat, LogOutputDest}; use sn_peers_acquisition::get_peers_from_args; use sn_peers_acquisition::PeersArgs; +use std::path::PathBuf; use tiny_http::{Response, Server}; #[derive(Parser)] @@ -34,6 +35,9 @@ struct Opt { /// Clear the local spend DAG and start from scratch #[clap(short, long)] clean: bool, + /// Visualize a local DAG file offline, does not connect to the Network + #[clap(short, long, value_name = "dag_file")] + offline_viewer: Option, /// Specify the logging output destination. /// @@ -62,6 +66,14 @@ async fn main() -> Result<()> { let opt = Opt::parse(); let log_builder = logging_init(opt.log_output_dest, opt.log_format)?; let _log_handles = log_builder.initialize()?; + + if let Some(dag_to_view) = opt.offline_viewer { + let dag = SpendDagDb::offline(dag_to_view)?; + dag.dump_dag_svg()?; + start_server(dag).await?; + return Ok(()); + } + let client = connect_to_network(opt.peers).await?; let dag = initialize_background_spend_dag_collection( client.clone(), diff --git a/sn_client/src/audit/spend_dag.rs b/sn_client/src/audit/spend_dag.rs index b11d99a424..b5bcfac819 100644 --- a/sn_client/src/audit/spend_dag.rs +++ b/sn_client/src/audit/spend_dag.rs @@ -75,7 +75,7 @@ pub enum SpendDagGet { /// Spend does not exist in the DAG SpendNotFound, /// Spend key is refered to by known spends but does not exist in the DAG yet - SpendKeyExists, + Utxo, /// Spend is a double spend DoubleSpend(Vec), /// Spend is in the DAG @@ -206,22 +206,17 @@ impl SpendDag { true } - /// Get the unknown parents: all the addresses that are refered to as parents by other spends - /// but don't have parents themselves. - /// Those Spends must exist somewhere on the Network, we just haven't gathered them yet. - pub fn get_unknown_parents(&self) -> BTreeSet { - let mut sources = BTreeSet::new(); - for node_index in self.dag.node_indices() { - if !self - .dag - .neighbors_directed(node_index, petgraph::Direction::Incoming) - .any(|_| true) - { - let utxo_addr = self.dag[node_index]; - sources.insert(utxo_addr); - } - } - sources + /// Get spend addresses that probably exist as they are refered to by spends we know, + /// but we haven't gathered them yet + /// This includes UTXOs and unknown ancestors + pub fn get_pending_spends(&self) -> BTreeSet { + self.spends + .iter() + .filter_map(|(addr, entry)| match entry { + DagEntry::NotGatheredYet(_) => Some(*addr), + _ => None, + }) + .collect() } /// Get the UTXOs: all the addresses that are refered to as children by other spends @@ -279,7 +274,7 @@ impl SpendDag { pub fn get_spend(&self, addr: &SpendAddress) -> SpendDagGet { match self.spends.get(addr) { None => SpendDagGet::SpendNotFound, - Some(DagEntry::NotGatheredYet(_)) => SpendDagGet::SpendKeyExists, + Some(DagEntry::NotGatheredYet(_)) => SpendDagGet::Utxo, Some(DagEntry::DoubleSpend(spends)) => { SpendDagGet::DoubleSpend(spends.iter().map(|(s, _)| s.clone()).collect()) } @@ -515,8 +510,20 @@ impl SpendDag { // record double spends if spends.len() > 1 { - debug!("Found a double spend entry in DAG: {source:?}"); + debug!("Found a double spend entry in DAG at {addr:?}"); recorded_faults.insert(SpendFault::DoubleSpend(*addr)); + let direct_descendants: BTreeSet = spends + .iter() + .flat_map(|s| s.spend.spent_tx.outputs.iter()) + .map(|o| SpendAddress::from_unique_pubkey(&o.unique_pubkey)) + .collect(); + debug!("Making the direct descendants of the double spend at {addr:?} as faulty: {direct_descendants:?}"); + for a in direct_descendants { + recorded_faults.insert(SpendFault::DoubleSpentAncestor { + addr: a, + ancestor: *addr, + }); + } continue; } @@ -599,13 +606,16 @@ impl SpendDag { let direct_descendants = spent_tx .outputs .iter() - .map(|o| SpendAddress::from_unique_pubkey(&o.unique_pubkey)); - let all_descendants = direct_descendants - .map(|addr| self.all_descendants(&addr)) + .map(|o| SpendAddress::from_unique_pubkey(&o.unique_pubkey)) + .collect::>(); + let mut all_descendants = direct_descendants + .iter() + .map(|addr| self.all_descendants(addr)) .collect::, _>>()? .into_iter() .flatten() .collect::>(); + all_descendants.extend(direct_descendants.iter()); for d in all_descendants { recorded_faults.insert(SpendFault::PoisonedAncestry(*d, poison.clone())); diff --git a/sn_client/src/audit/tests/mod.rs b/sn_client/src/audit/tests/mod.rs index 5a9d28149e..b0c475e774 100644 --- a/sn_client/src/audit/tests/mod.rs +++ b/sn_client/src/audit/tests/mod.rs @@ -13,6 +13,7 @@ use std::collections::BTreeSet; use setup::MockNetwork; use eyre::Result; +use sn_transfers::SpendAddress; use crate::{SpendDag, SpendFault}; @@ -38,11 +39,94 @@ fn test_spend_dag_verify_valid_simple() -> Result<()> { for spend in net.spends { dag.insert(spend.address(), spend.clone()); } + assert!(dag.record_faults(&genesis).is_ok()); + // dag.dump_to_file("/tmp/test_spend_dag_verify_valid_simple")?; assert_eq!(dag.verify(&genesis), Ok(BTreeSet::new())); Ok(()) } +#[test] +fn test_spend_dag_double_spend_poisonning() -> Result<()> { + let mut net = MockNetwork::genesis()?; + let genesis = net.genesis_spend; + + let owner1 = net.new_pk_with_balance(100)?; + let owner2 = net.new_pk_with_balance(0)?; + let owner3 = net.new_pk_with_balance(0)?; + let owner4 = net.new_pk_with_balance(0)?; + let owner5 = net.new_pk_with_balance(0)?; + let owner6 = net.new_pk_with_balance(0)?; + let owner_cheat = net.new_pk_with_balance(0)?; + + // spend normaly and save a cashnote to reuse later + net.send(&owner1, &owner2, 100)?; + let cn_to_reuse_later = net + .wallets + .get(&owner2) + .expect("owner2 wallet to exist") + .cn + .clone(); + let spend1 = net.send(&owner2, &owner3, 100)?; + let spend_ko3 = net.send(&owner3, &owner4, 100)?; + let spend_ok4 = net.send(&owner4, &owner5, 100)?; + let spend_ok5 = net.send(&owner5, &owner6, 100)?; + + // reuse that cashnote to perform a double spend far back in history + net.wallets + .get_mut(&owner2) + .expect("owner2 wallet to still exist") + .cn = cn_to_reuse_later; + let spend2 = net.send(&owner2, &owner_cheat, 100)?; + + // create dag + let mut dag = SpendDag::new(genesis); + for spend in net.spends { + dag.insert(spend.address(), spend.clone()); + } + assert!(dag.record_faults(&genesis).is_ok()); + // dag.dump_to_file("/tmp/test_spend_dag_double_spend_poisonning")?; + + // make sure double spend is detected + assert_eq!(spend1, spend2, "both spends should be at the same address"); + let double_spent = spend1.first().expect("spend1 to have an element"); + let got = dag.get_spend_faults(double_spent); + let expected = BTreeSet::from_iter([SpendFault::DoubleSpend(*double_spent)]); + assert_eq!(got, expected, "DAG should have detected double spend"); + + // make sure the double spend's direct descendants are unspendable + let upk = net + .wallets + .get(&owner_cheat) + .expect("owner_cheat wallet to exist") + .cn + .first() + .expect("owner_cheat wallet to have 1 cashnote") + .unique_pubkey(); + let utxo = SpendAddress::from_unique_pubkey(&upk); + let got = dag.get_spend_faults(&utxo); + let expected = BTreeSet::from_iter([SpendFault::DoubleSpentAncestor { + addr: utxo, + ancestor: *double_spent, + }]); + assert_eq!(got, expected, "UTXO of double spend should be unspendable"); + let s3 = spend_ko3.first().expect("spend_ko3 to have an element"); + let got = dag.get_spend_faults(s3); + let expected = BTreeSet::from_iter([SpendFault::DoubleSpentAncestor { + addr: *s3, + ancestor: *double_spent, + }]); + assert_eq!(got, expected, "spend_ko3 should be unspendable"); + + // make sure this didn't affect the rest of the DAG + let s4 = spend_ok4.first().expect("spend_ok4 to be unique"); + let s5 = spend_ok5.first().expect("spend_ok5 to be unique"); + + assert_eq!(dag.get_spend_faults(s4), BTreeSet::new()); + assert_eq!(dag.get_spend_faults(s5), BTreeSet::new()); + Ok(()) +} + #[test] fn test_spend_dag_double_spend_detection() -> Result<()> { let mut net = MockNetwork::genesis()?; @@ -52,6 +136,7 @@ fn test_spend_dag_double_spend_detection() -> Result<()> { let owner2a = net.new_pk_with_balance(0)?; let owner2b = net.new_pk_with_balance(0)?; + // perform double spend let cn_to_reuse = net .wallets .get(&owner1) @@ -65,12 +150,35 @@ fn test_spend_dag_double_spend_detection() -> Result<()> { .cn = cn_to_reuse; let spend2_addr = net.send(&owner1, &owner2b, 100)?; + // get the UTXOs of the two spends + let upk_of_2a = net + .wallets + .get(&owner2a) + .expect("owner2a wallet to exist") + .cn + .first() + .expect("owner2a wallet to have 1 cashnote") + .unique_pubkey(); + let utxo_of_2a = SpendAddress::from_unique_pubkey(&upk_of_2a); + let upk_of_2b = net + .wallets + .get(&owner2b) + .expect("owner2b wallet to exist") + .cn + .first() + .expect("owner2b wallet to have 1 cashnote") + .unique_pubkey(); + let utxo_of_2b = SpendAddress::from_unique_pubkey(&upk_of_2b); + + // make DAG let mut dag = SpendDag::new(genesis); for spend in net.spends { dag.insert(spend.address(), spend.clone()); } dag.record_faults(&genesis)?; + // dag.dump_to_file("/tmp/test_spend_dag_double_spend_detection")?; + // make sure the double spend is detected assert_eq!( spend1_addr, spend2_addr, "both spends should be at the same address" @@ -83,5 +191,184 @@ fn test_spend_dag_double_spend_detection() -> Result<()> { expected, "DAG should have detected double spend" ); + + // make sure the UTXOs of the double spend are unspendable + let got = dag.get_spend_faults(&utxo_of_2a); + let expected = BTreeSet::from_iter([SpendFault::DoubleSpentAncestor { + addr: utxo_of_2a, + ancestor: *double_spent, + }]); + assert_eq!( + got, expected, + "UTXO a of double spend should be unspendable" + ); + + let got = dag.get_spend_faults(&utxo_of_2b); + let expected = BTreeSet::from_iter([SpendFault::DoubleSpentAncestor { + addr: utxo_of_2b, + ancestor: *double_spent, + }]); + assert_eq!( + got, expected, + "UTXO b of double spend should be unspendable" + ); + Ok(()) +} + +#[test] +fn test_spend_dag_missing_ancestry() -> Result<()> { + let mut net = MockNetwork::genesis()?; + let genesis = net.genesis_spend; + + let owner1 = net.new_pk_with_balance(100)?; + let owner2 = net.new_pk_with_balance(0)?; + let owner3 = net.new_pk_with_balance(0)?; + let owner4 = net.new_pk_with_balance(0)?; + let owner5 = net.new_pk_with_balance(0)?; + let owner6 = net.new_pk_with_balance(0)?; + + net.send(&owner1, &owner2, 100)?; + net.send(&owner2, &owner3, 100)?; + let spend_missing = net + .send(&owner3, &owner4, 100)? + .first() + .expect("spend_missing should have 1 element") + .to_owned(); + let spent_after1 = net + .send(&owner4, &owner5, 100)? + .first() + .expect("spent_after1 should have 1 element") + .to_owned(); + let spent_after2 = net + .send(&owner5, &owner6, 100)? + .first() + .expect("spent_after2 should have 1 element") + .to_owned(); + let utxo_after3 = net + .wallets + .get(&owner6) + .expect("owner6 wallet to exist") + .cn + .first() + .expect("owner6 wallet to have 1 cashnote") + .unique_pubkey(); + let utxo_addr = SpendAddress::from_unique_pubkey(&utxo_after3); + + // create dag with one missing spend + let net_spends = net + .spends + .into_iter() + .filter(|s| spend_missing != s.address()); + let mut dag = SpendDag::new(genesis); + for spend in net_spends { + dag.insert(spend.address(), spend.clone()); + } + dag.record_faults(&genesis)?; + // dag.dump_to_file("/tmp/test_spend_dag_missing_ancestry")?; + + // make sure the missing spend makes its descendants invalid + let got = dag.get_spend_faults(&spent_after1); + let expected = BTreeSet::from_iter([SpendFault::MissingAncestry { + addr: spent_after1, + ancestor: spend_missing, + }]); + assert_eq!(got, expected, "DAG should have detected missing ancestry"); + + let got = dag.get_spend_faults(&spent_after2); + let expected = BTreeSet::from_iter([SpendFault::PoisonedAncestry( + spent_after2, + format!("missing ancestor at: {spend_missing:?}"), + )]); + assert_eq!( + got, expected, + "DAG should have propagated the error to descendants" + ); + + let got = dag.get_spend_faults(&utxo_addr); + let expected = BTreeSet::from_iter([SpendFault::PoisonedAncestry( + utxo_addr, + format!("missing ancestor at: {spend_missing:?}"), + )]); + assert_eq!( + got, expected, + "DAG should have propagated the error all the way to descendant utxos" + ); + Ok(()) +} + +#[test] +fn test_spend_dag_orphans() -> Result<()> { + let mut net = MockNetwork::genesis()?; + let genesis = net.genesis_spend; + + let owner1 = net.new_pk_with_balance(100)?; + let owner2 = net.new_pk_with_balance(0)?; + let owner3 = net.new_pk_with_balance(0)?; + let owner4 = net.new_pk_with_balance(0)?; + let owner5 = net.new_pk_with_balance(0)?; + let owner6 = net.new_pk_with_balance(0)?; + + net.send(&owner1, &owner2, 100)?; + net.send(&owner2, &owner3, 100)?; + let spend_missing1 = net + .send(&owner3, &owner4, 100)? + .first() + .expect("spend_missing should have 1 element") + .to_owned(); + let spend_missing2 = net + .send(&owner4, &owner5, 100)? + .first() + .expect("spend_missing2 should have 1 element") + .to_owned(); + let spent_after1 = net + .send(&owner5, &owner6, 100)? + .first() + .expect("spent_after1 should have 1 element") + .to_owned(); + let utxo_after2 = net + .wallets + .get(&owner6) + .expect("owner6 wallet to exist") + .cn + .first() + .expect("owner6 wallet to have 1 cashnote") + .unique_pubkey(); + let utxo_addr = SpendAddress::from_unique_pubkey(&utxo_after2); + + // create dag with two missing spends in the chain + let net_spends = net + .spends + .into_iter() + .filter(|s| spend_missing1 != s.address() && spend_missing2 != s.address()); + let mut dag = SpendDag::new(genesis); + for spend in net_spends { + dag.insert(spend.address(), spend.clone()); + } + dag.record_faults(&genesis)?; + // dag.dump_to_file("/tmp/test_spend_dag_orphans")?; + + // make sure the spends after the two missing spends are orphans + let got = dag.get_spend_faults(&spent_after1); + let expected = BTreeSet::from_iter([ + SpendFault::OrphanSpend { + addr: spent_after1, + src: dag.source(), + }, + SpendFault::MissingAncestry { + addr: spent_after1, + ancestor: spend_missing2, + }, + ]); + assert_eq!(got, expected, "DAG should have detected orphan spend"); + + let got = dag.get_spend_faults(&utxo_addr); + let expected = BTreeSet::from_iter([SpendFault::OrphanSpend { + addr: utxo_addr, + src: dag.source(), + }]); + assert_eq!( + got, expected, + "Utxo of orphan spend should also be an orphan" + ); Ok(()) } From 8422ebd945d75df3b379d230468ed8c6856806f3 Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 11 Apr 2024 18:24:28 +0900 Subject: [PATCH 051/205] fix: orphan test --- sn_client/src/audit/tests/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sn_client/src/audit/tests/mod.rs b/sn_client/src/audit/tests/mod.rs index b0c475e774..2f4933d4cd 100644 --- a/sn_client/src/audit/tests/mod.rs +++ b/sn_client/src/audit/tests/mod.rs @@ -362,12 +362,12 @@ fn test_spend_dag_orphans() -> Result<()> { assert_eq!(got, expected, "DAG should have detected orphan spend"); let got = dag.get_spend_faults(&utxo_addr); - let expected = BTreeSet::from_iter([SpendFault::OrphanSpend { + let expected = SpendFault::OrphanSpend { addr: utxo_addr, src: dag.source(), - }]); - assert_eq!( - got, expected, + }; + assert!( + got.contains(&expected), "Utxo of orphan spend should also be an orphan" ); Ok(()) From cef9855ce5a401ee57d352db64961125976ba19e Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Thu, 11 Apr 2024 15:39:51 +0530 Subject: [PATCH 052/205] feat(store): load existing records in parallel --- sn_networking/src/record_store.rs | 109 +++++++++++++++--------------- 1 file changed, 56 insertions(+), 53 deletions(-) diff --git a/sn_networking/src/record_store.rs b/sn_networking/src/record_store.rs index 24004ba5ca..88eccb2079 100644 --- a/sn_networking/src/record_store.rs +++ b/sn_networking/src/record_store.rs @@ -15,6 +15,7 @@ use aes_gcm_siv::{ Aes256GcmSiv, Nonce, }; +use itertools::Itertools; use libp2p::{ identity::PeerId, kad::{ @@ -25,6 +26,7 @@ use libp2p::{ #[cfg(feature = "open-metrics")] use prometheus_client::metrics::gauge::Gauge; use rand::RngCore; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; use sn_protocol::{ storage::{RecordHeader, RecordKind, RecordType}, @@ -40,7 +42,7 @@ use std::{ vec, }; use tokio::sync::mpsc; -use walkdir::WalkDir; +use walkdir::{DirEntry, WalkDir}; use xor_name::XorName; /// Max number of records a node can store @@ -122,67 +124,68 @@ impl NodeRecordStore { config: &NodeRecordStoreConfig, encryption_details: &(Aes256GcmSiv, [u8; 4]), ) -> HashMap { - let mut records = HashMap::default(); - - info!("Attempting to repopulate records from existing store..."); - for entry in WalkDir::new(&config.storage_dir) - .into_iter() - .filter_map(|e| e.ok()) - { + let process_entry = |entry: &DirEntry| -> _ { let path = entry.path(); if path.is_file() { info!("Existing record found: {path:?}"); // if we've got a file, lets try and read it - if let Some(filename) = path.file_name().and_then(|n| n.to_str()) { - // get the record key from the filename - if let Some(key) = Self::get_data_from_filename(filename) { - let record = match fs::read(path) { - Ok(bytes) => { - // and the stored record - Self::get_record_from_bytes(bytes, &key, encryption_details) - } - Err(err) => { - error!( - "Error while reading file. filename: {filename}, error: {err:?}" - ); - None - } - }; - - if let Some(record) = record { - let record_type = match RecordHeader::is_record_of_type_chunk(&record) { - Ok(true) => RecordType::Chunk, - Ok(false) => { - let xorname_hash = XorName::from_content(&record.value); - RecordType::NonChunk(xorname_hash) - } - Err(error) => { - warn!("Failed to parse record type from record: {:?}", error); - continue; - } - }; - - let address = NetworkAddress::from_record_key(&key); - records.insert(key, (address, record_type)); - info!("Existing record loaded: {path:?}"); - } - }; - } else { - // warn and remove this file as it's not a valid record - warn!( - "Found a file in the storage dir that is not a valid record: {:?}", - path - ); - if let Err(e) = fs::remove_file(path) { + let filename = match path.file_name().and_then(|n| n.to_str()) { + Some(file_name) => file_name, + None => { + // warn and remove this file as it's not a valid record warn!( - "Failed to remove invalid record file from storage dir: {:?}", - e + "Found a file in the storage dir that is not a valid record: {:?}", + path ); + if let Err(e) = fs::remove_file(path) { + warn!( + "Failed to remove invalid record file from storage dir: {:?}", + e + ); + } + return None; } - } + }; + // get the record key from the filename + let key = Self::get_data_from_filename(filename)?; + let record = match fs::read(path) { + Ok(bytes) => { + // and the stored record + Self::get_record_from_bytes(bytes, &key, encryption_details)? + } + Err(err) => { + error!("Error while reading file. filename: {filename}, error: {err:?}"); + return None; + } + }; + + let record_type = match RecordHeader::is_record_of_type_chunk(&record) { + Ok(true) => RecordType::Chunk, + Ok(false) => { + let xorname_hash = XorName::from_content(&record.value); + RecordType::NonChunk(xorname_hash) + } + Err(error) => { + warn!("Failed to parse record type from record: {:?}", error); + return None; + } + }; + + let address = NetworkAddress::from_record_key(&key); + info!("Existing record loaded: {path:?}"); + return Some((key, (address, record_type))); } - } + None + }; + info!("Attempting to repopulate records from existing store..."); + let records = WalkDir::new(&config.storage_dir) + .into_iter() + .filter_map(|e| e.ok()) + .collect_vec() + .par_iter() + .filter_map(process_entry) + .collect(); records } From 7c5d931b2be1f749143cdfed2ed77f426e7dfc1a Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Thu, 11 Apr 2024 15:24:41 +0200 Subject: [PATCH 053/205] refactor(networking): remove circular vec error --- sn_networking/src/circular_vec.rs | 25 ++++++++++++------------- sn_networking/src/error.rs | 3 --- sn_networking/src/event.rs | 8 ++------ 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/sn_networking/src/circular_vec.rs b/sn_networking/src/circular_vec.rs index e0c1d7786a..0ef3aa0d24 100644 --- a/sn_networking/src/circular_vec.rs +++ b/sn_networking/src/circular_vec.rs @@ -6,37 +6,34 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::error::NetworkError; - /// Based on https://users.rust-lang.org/t/the-best-ring-buffer-library/58489/7 /// A circular buffer implemented with a VecDeque. #[derive(Debug)] -pub struct CircularVec { +pub(crate) struct CircularVec { inner: std::collections::VecDeque, } impl CircularVec { /// Creates a new CircularVec with the given capacity. - pub fn new(capacity: usize) -> Self { + /// + /// Capacity is normally rounded up to the nearest power of 2, minus one. E.g. 15, 31, 63, 127, 255, etc. + pub(crate) fn new(capacity: usize) -> Self { Self { inner: std::collections::VecDeque::with_capacity(capacity), } } /// Pushes an item into the CircularVec. If the CircularVec is full, the oldest item is removed. - pub fn push(&mut self, item: T) -> Result<(), NetworkError> { + pub(crate) fn push(&mut self, item: T) { if self.inner.len() == self.inner.capacity() { - self.inner - .pop_front() - .ok_or(NetworkError::CircularVecPopFrontError)?; + let _ = self.inner.pop_front(); } self.inner.push_back(item); - Ok(()) } /// Checks if the CircularVec contains the given item. - pub fn contains(&self, item: &T) -> bool + pub(crate) fn contains(&self, item: &T) -> bool where T: PartialEq, { @@ -51,14 +48,16 @@ mod tests { #[test] fn test_push_and_contains() { let mut cv = CircularVec::new(2); - assert!(cv.push(1).is_ok()); - assert!(cv.push(2).is_ok()); + cv.push(1); + cv.push(2); assert!(cv.contains(&1)); assert!(cv.contains(&2)); - assert!(cv.push(3).is_ok()); + cv.push(3); assert!(!cv.contains(&1)); assert!(cv.contains(&2)); assert!(cv.contains(&3)); + + assert!(cv.inner.len() == 2); } } diff --git a/sn_networking/src/error.rs b/sn_networking/src/error.rs index 978bcb5f1a..841c059186 100644 --- a/sn_networking/src/error.rs +++ b/sn_networking/src/error.rs @@ -158,9 +158,6 @@ pub enum NetworkError { #[error("Close group size must be a non-zero usize")] InvalidCloseGroupSize, - #[error("Failed to pop from front of CircularVec")] - CircularVecPopFrontError, - #[error("Node Listen Address was not provided during construction")] ListenAddressNotProvided, diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 9c777bb8ff..bc13430e8b 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -309,9 +309,7 @@ impl SwarmDriver { if !kbucket_full { info!(%peer_id, ?addrs, "received identify info from undialed peer for not full kbucket {ilog2:?}, dial back to confirm external accessible"); - self.dialed_peers - .push(peer_id) - .map_err(|_| NetworkError::CircularVecPopFrontError)?; + self.dialed_peers.push(peer_id); if let Err(err) = self.swarm.dial( DialOpts::peer_id(peer_id) .condition(PeerCondition::NotDialing) @@ -444,9 +442,7 @@ impl SwarmDriver { ); if endpoint.is_dialer() { - self.dialed_peers - .push(peer_id) - .map_err(|_| NetworkError::CircularVecPopFrontError)?; + self.dialed_peers.push(peer_id); } } SwarmEvent::ConnectionClosed { From 592ac510efa3741ea778ec6b994318ac7ef79470 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Thu, 11 Apr 2024 18:06:42 +0100 Subject: [PATCH 054/205] feat: distinguish failure to start during upgrade It's possible for a service to be upgraded but then not subsequently start. In this case, it has still been upgraded to a new version. It's worth making a distinction to the user between an actual error in the upgrade process, or a failure to start, because in the latter case, they do actually have an upgrade. They can then take action to try and start their services again. As part of this change, the start process attempts to find whether the service process did indeed start, because you don't always seem to get errors back from the service infrastructure. We also make sure that we return an error if there was a failure with the upgrade process for any services. This is necessary for visibility on our own deploy process. --- sn_node_manager/src/add_services/tests.rs | 4 +- sn_node_manager/src/cmd/mod.rs | 7 + sn_node_manager/src/cmd/node.rs | 15 +- sn_node_manager/src/lib.rs | 256 +++++++++++++++++++++- sn_service_management/src/control.rs | 23 +- sn_service_management/src/daemon.rs | 2 +- sn_service_management/src/error.rs | 2 +- sn_service_management/src/faucet.rs | 2 +- sn_service_management/src/lib.rs | 1 + 9 files changed, 289 insertions(+), 23 deletions(-) diff --git a/sn_node_manager/src/add_services/tests.rs b/sn_node_manager/src/add_services/tests.rs index b7fab7bdc5..b8497b68d2 100644 --- a/sn_node_manager/src/add_services/tests.rs +++ b/sn_node_manager/src/add_services/tests.rs @@ -31,7 +31,7 @@ use sn_service_management::{ use std::{ ffi::OsString, net::{IpAddr, Ipv4Addr, SocketAddr}, - path::PathBuf, + path::{Path, PathBuf}, str::FromStr, }; @@ -54,7 +54,7 @@ mock! { fn create_service_user(&self, username: &str) -> ServiceControlResult<()>; fn get_available_port(&self) -> ServiceControlResult; fn install(&self, install_ctx: ServiceInstallCtx) -> ServiceControlResult<()>; - fn get_process_pid(&self, name: &str) -> ServiceControlResult; + fn get_process_pid(&self, bin_path: &Path) -> ServiceControlResult; fn is_service_process_running(&self, pid: u32) -> bool; fn start(&self, service_name: &str) -> ServiceControlResult<()>; fn stop(&self, service_name: &str) -> ServiceControlResult<()>; diff --git a/sn_node_manager/src/cmd/mod.rs b/sn_node_manager/src/cmd/mod.rs index 4634d3b09d..07e38f8c83 100644 --- a/sn_node_manager/src/cmd/mod.rs +++ b/sn_node_manager/src/cmd/mod.rs @@ -75,6 +75,13 @@ pub fn print_upgrade_summary(upgrade_summary: Vec<(String, UpgradeResult)>) { service_name ); } + UpgradeResult::UpgradedButNotStarted(previous_version, new_version, _) => { + println!( + "{} {} was upgraded from {previous_version} to {new_version} but it did not start", + "✕".red(), + service_name + ); + } UpgradeResult::Forced(previous_version, target_version) => { println!( "{} Forced {} version change from {previous_version} to {target_version}.", diff --git a/sn_node_manager/src/cmd/node.rs b/sn_node_manager/src/cmd/node.rs index 16c8ed6eb6..a8352752cb 100644 --- a/sn_node_manager/src/cmd/node.rs +++ b/sn_node_manager/src/cmd/node.rs @@ -18,7 +18,7 @@ use crate::{ helpers::{download_and_extract_release, get_bin_version}, refresh_node_registry, status_report, ServiceManager, VerbosityLevel, }; -use color_eyre::{eyre::eyre, Result}; +use color_eyre::{eyre::eyre, Help, Result}; use colored::Colorize; use libp2p_identity::PeerId; use semver::Version; @@ -358,9 +358,18 @@ pub async fn upgrade( } } - print_upgrade_summary(upgrade_summary); - node_registry.save()?; + print_upgrade_summary(upgrade_summary.clone()); + + if upgrade_summary.iter().any(|(_, r)| { + matches!(r, UpgradeResult::Error(_)) + || matches!(r, UpgradeResult::UpgradedButNotStarted(_, _, _)) + }) { + return Err(eyre!("There was a problem upgrading one or more nodes").suggestion( + "For any services that were upgraded but did not start, you can attempt to start them \ + again using the 'start' command.")); + } + Ok(()) } diff --git a/sn_node_manager/src/lib.rs b/sn_node_manager/src/lib.rs index b60860d75e..c16c6016f7 100644 --- a/sn_node_manager/src/lib.rs +++ b/sn_node_manager/src/lib.rs @@ -44,6 +44,7 @@ use sn_service_management::{ NodeRegistry, NodeServiceData, ServiceStateActions, ServiceStatus, UpgradeOptions, UpgradeResult, }; +use tracing::debug; pub const DAEMON_DEFAULT_PORT: u16 = 12500; pub const DAEMON_SERVICE_NAME: &str = "safenodemand"; @@ -91,6 +92,31 @@ impl ServiceManager { self.service_control.start(&self.service.name())?; self.service_control.wait(RPC_START_UP_DELAY_MS); + // This is an attempt to see whether the service process has actually launched. You don't + // always get an error from the service infrastructure. + // + // There might be many different `safenode` processes running, but since each service has + // its own isolated binary, we use the binary path to uniquely identify it. + match self + .service_control + .get_process_pid(&self.service.bin_path()) + { + Ok(pid) => { + debug!( + "Service process started for {} with PID {}", + self.service.name(), + pid + ); + } + Err(sn_service_management::error::Error::ServiceProcessNotFound(_)) => { + return Err(eyre!( + "The '{}' service has failed to start", + self.service.name() + )); + } + Err(e) => return Err(e.into()), + } + self.service.on_start().await?; println!("{} Started {} service", "✓".green(), self.service.name()); @@ -236,7 +262,18 @@ impl ServiceManager { )?; if options.start_service { - self.start().await?; + match self.start().await { + Ok(()) => {} + Err(e) => { + self.service + .set_version(&options.target_version.to_string()); + return Ok(UpgradeResult::UpgradedButNotStarted( + current_version.to_string(), + options.target_version.to_string(), + e.to_string(), + )); + } + } } self.service .set_version(&options.target_version.to_string()); @@ -444,14 +481,14 @@ mod tests { use predicates::prelude::*; use service_manager::ServiceInstallCtx; use sn_service_management::{ - error::Result as ServiceControlResult, + error::{Error as ServiceControlError, Result as ServiceControlResult}, node::{NodeService, NodeServiceData}, rpc::{NetworkInfo, NodeInfo, RecordAddress, RpcActions}, UpgradeOptions, UpgradeResult, }; use std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, - path::PathBuf, + path::{Path, PathBuf}, str::FromStr, }; @@ -475,7 +512,7 @@ mod tests { fn create_service_user(&self, username: &str) -> ServiceControlResult<()>; fn get_available_port(&self) -> ServiceControlResult; fn install(&self, install_ctx: ServiceInstallCtx) -> ServiceControlResult<()>; - fn get_process_pid(&self, name: &str) -> ServiceControlResult; + fn get_process_pid(&self, bin_path: &Path) -> ServiceControlResult; fn is_service_process_running(&self, pid: u32) -> bool; fn start(&self, service_name: &str) -> ServiceControlResult<()>; fn stop(&self, service_name: &str) -> ServiceControlResult<()>; @@ -499,6 +536,13 @@ mod tests { .with(eq(3000)) .times(1) .returning(|_| ()); + mock_service_control + .expect_get_process_pid() + .with(eq(PathBuf::from( + "/var/safenode-manager/services/safenode1/safenode", + ))) + .times(1) + .returning(|_| Ok(100)); mock_rpc_client.expect_node_info().times(1).returning(|| { Ok(NodeInfo { pid: 1000, @@ -575,6 +619,13 @@ mod tests { .with(eq(3000)) .times(1) .returning(|_| ()); + mock_service_control + .expect_get_process_pid() + .with(eq(PathBuf::from( + "/var/safenode-manager/services/safenode1/safenode", + ))) + .times(1) + .returning(|_| Ok(100)); mock_rpc_client.expect_node_info().times(1).returning(|| { Ok(NodeInfo { pid: 1000, @@ -697,6 +748,11 @@ mod tests { let mut mock_service_control = MockServiceControl::new(); let mut mock_rpc_client = MockRpcClient::new(); + mock_service_control + .expect_is_service_process_running() + .with(eq(1000)) + .times(1) + .returning(|_| false); mock_service_control .expect_start() .with(eq("safenode1")) @@ -708,10 +764,12 @@ mod tests { .times(1) .returning(|_| ()); mock_service_control - .expect_is_service_process_running() - .with(eq(1000)) + .expect_get_process_pid() + .with(eq(PathBuf::from( + "/var/safenode-manager/services/safenode1/safenode", + ))) .times(1) - .returning(|_| false); + .returning(|_| Ok(100)); mock_rpc_client.expect_node_info().times(1).returning(|| { Ok(NodeInfo { pid: 1000, @@ -775,6 +833,65 @@ mod tests { Ok(()) } + #[tokio::test] + async fn start_should_return_an_error_if_the_process_was_not_found() -> Result<()> { + let mut mock_service_control = MockServiceControl::new(); + + mock_service_control + .expect_start() + .with(eq("safenode1")) + .times(1) + .returning(|_| Ok(())); + mock_service_control + .expect_wait() + .with(eq(3000)) + .times(1) + .returning(|_| ()); + mock_service_control + .expect_get_process_pid() + .with(eq(PathBuf::from( + "/var/safenode-manager/services/safenode1/safenode", + ))) + .times(1) + .returning(|_| { + Err(ServiceControlError::ServiceProcessNotFound( + "/var/safenode-manager/services/safenode1/safenode".to_string(), + )) + }); + + let mut service_data = NodeServiceData { + connected_peers: None, + data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), + genesis: false, + listen_addr: None, + local: false, + log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), + number: 1, + peer_id: None, + pid: None, + rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), + safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), + service_name: "safenode1".to_string(), + status: ServiceStatus::Added, + user: "safe".to_string(), + version: "0.98.1".to_string(), + }; + let service = NodeService::new(&mut service_data, Box::new(MockRpcClient::new())); + let mut service_manager = ServiceManager::new( + service, + Box::new(mock_service_control), + VerbosityLevel::Normal, + ); + + let result = service_manager.start().await; + match result { + Ok(_) => panic!("This test should have resulted in an error"), + Err(e) => assert_eq!("The 'safenode1' service has failed to start", e.to_string()), + } + + Ok(()) + } + #[tokio::test] async fn stop_should_stop_a_running_service() -> Result<()> { let mut mock_service_control = MockServiceControl::new(); @@ -994,6 +1111,11 @@ mod tests { .with(eq(3000)) .times(1) .returning(|_| ()); + mock_service_control + .expect_get_process_pid() + .with(eq(current_node_bin.to_path_buf().clone())) + .times(1) + .returning(|_| Ok(100)); mock_rpc_client.expect_node_info().times(1).returning(|| { Ok(NodeInfo { pid: 2000, @@ -1185,6 +1307,11 @@ mod tests { .with(eq(3000)) .times(1) .returning(|_| ()); + mock_service_control + .expect_get_process_pid() + .with(eq(current_node_bin.to_path_buf().clone())) + .times(1) + .returning(|_| Ok(100)); mock_rpc_client.expect_node_info().times(1).returning(|| { Ok(NodeInfo { pid: 2000, @@ -1401,6 +1528,121 @@ mod tests { Ok(()) } + #[tokio::test] + async fn upgrade_should_return_upgraded_but_not_started_if_service_did_not_start() -> Result<()> + { + let current_version = "0.1.0"; + let target_version = "0.2.0"; + + let tmp_data_dir = assert_fs::TempDir::new()?; + let current_install_dir = tmp_data_dir.child("safenode_install"); + current_install_dir.create_dir_all()?; + + let current_node_bin = current_install_dir.child("safenode"); + current_node_bin.write_binary(b"fake safenode binary")?; + let target_node_bin = tmp_data_dir.child("safenode"); + target_node_bin.write_binary(b"fake safenode binary")?; + + let current_node_bin_str = current_node_bin.to_path_buf().to_string_lossy().to_string(); + + let mut mock_service_control = MockServiceControl::new(); + + // before binary upgrade + mock_service_control + .expect_is_service_process_running() + .with(eq(1000)) + .times(1) + .returning(|_| true); + mock_service_control + .expect_stop() + .with(eq("safenode1")) + .times(1) + .returning(|_| Ok(())); + + // after binary upgrade + mock_service_control + .expect_uninstall() + .with(eq("safenode1")) + .times(1) + .returning(|_| Ok(())); + mock_service_control + .expect_install() + .with(always()) + .times(1) + .returning(|_| Ok(())); + + // after service restart + mock_service_control + .expect_start() + .with(eq("safenode1")) + .times(1) + .returning(|_| Ok(())); + mock_service_control + .expect_wait() + .with(eq(3000)) + .times(1) + .returning(|_| ()); + mock_service_control + .expect_get_process_pid() + .with(eq(current_node_bin.to_path_buf().clone())) + .times(1) + .returning(move |_| { + Err(ServiceControlError::ServiceProcessNotFound( + current_node_bin_str.clone(), + )) + }); + + let mut service_data = NodeServiceData { + connected_peers: None, + data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), + genesis: false, + listen_addr: None, + local: false, + log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), + number: 1, + peer_id: Some(PeerId::from_str( + "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", + )?), + pid: Some(1000), + rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), + safenode_path: current_node_bin.to_path_buf(), + service_name: "safenode1".to_string(), + status: ServiceStatus::Running, + user: "safe".to_string(), + version: current_version.to_string(), + }; + let service = NodeService::new(&mut service_data, Box::new(MockRpcClient::new())); + let mut service_manager = ServiceManager::new( + service, + Box::new(mock_service_control), + VerbosityLevel::Normal, + ); + + let upgrade_result = service_manager + .upgrade(UpgradeOptions { + bootstrap_peers: Vec::new(), + env_variables: None, + force: false, + start_service: true, + target_bin_path: target_node_bin.to_path_buf(), + target_version: Version::parse(target_version).unwrap(), + }) + .await?; + + match upgrade_result { + UpgradeResult::UpgradedButNotStarted(old_version, new_version, _) => { + assert_eq!(old_version, current_version); + assert_eq!(new_version, target_version); + } + _ => panic!( + "Expected UpgradeResult::UpgradedButNotStarted but was {:#?}", + upgrade_result + ), + } + + Ok(()) + } + #[tokio::test] async fn remove_should_remove_an_added_node() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; diff --git a/sn_service_management/src/control.rs b/sn_service_management/src/control.rs index 11bc2e7218..8a8a6f5565 100644 --- a/sn_service_management/src/control.rs +++ b/sn_service_management/src/control.rs @@ -12,7 +12,10 @@ use service_manager::{ ServiceInstallCtx, ServiceLabel, ServiceManager, ServiceStartCtx, ServiceStopCtx, ServiceUninstallCtx, }; -use std::net::{SocketAddr, TcpListener}; +use std::{ + net::{SocketAddr, TcpListener}, + path::Path, +}; use sysinfo::{Pid, System}; /// A thin wrapper around the `service_manager::ServiceManager`, which makes our own testing @@ -26,7 +29,7 @@ pub trait ServiceControl: Sync { fn create_service_user(&self, username: &str) -> Result<()>; fn get_available_port(&self) -> Result; fn install(&self, install_ctx: ServiceInstallCtx) -> Result<()>; - fn get_process_pid(&self, name: &str) -> Result; + fn get_process_pid(&self, path: &Path) -> Result; fn is_service_process_running(&self, pid: u32) -> bool; fn start(&self, service_name: &str) -> Result<()>; fn stop(&self, service_name: &str) -> Result<()>; @@ -161,17 +164,21 @@ impl ServiceControl for ServiceController { Ok(port) } - fn get_process_pid(&self, name: &str) -> Result { + fn get_process_pid(&self, bin_path: &Path) -> Result { let mut system = System::new_all(); system.refresh_all(); for (pid, process) in system.processes() { - if process.name() == name { - // There does not seem to be any easy way to get the process ID from the `Pid` - // type. Probably something to do with representing it in a cross-platform way. - return Ok(pid.to_string().parse::()?); + if let Some(path) = process.exe() { + if bin_path == path { + // There does not seem to be any easy way to get the process ID from the `Pid` + // type. Probably something to do with representing it in a cross-platform way. + return Ok(pid.to_string().parse::()?); + } } } - Err(Error::ServiceProcessNotFound(name.to_string())) + Err(Error::ServiceProcessNotFound( + bin_path.to_string_lossy().to_string(), + )) } fn install(&self, install_ctx: ServiceInstallCtx) -> Result<()> { diff --git a/sn_service_management/src/daemon.rs b/sn_service_management/src/daemon.rs index 72c8da6135..4822303e60 100644 --- a/sn_service_management/src/daemon.rs +++ b/sn_service_management/src/daemon.rs @@ -96,7 +96,7 @@ impl<'a> ServiceStateActions for DaemonService<'a> { // get_process_pid causes errors for the daemon. Maybe because it is being run as root? if let Ok(pid) = self .service_control - .get_process_pid(&self.service_data.service_name) + .get_process_pid(&self.service_data.daemon_path) { self.service_data.pid = Some(pid); } diff --git a/sn_service_management/src/error.rs b/sn_service_management/src/error.rs index 354e5ea864..726775ee69 100644 --- a/sn_service_management/src/error.rs +++ b/sn_service_management/src/error.rs @@ -43,7 +43,7 @@ pub enum Error { RpcNodeUpdateError(String), #[error("Could not obtain record addresses through RPC: {0}")] RpcRecordAddressError(String), - #[error("Could not find process '{0}'")] + #[error("Could not find process at '{0}'")] ServiceProcessNotFound(String), #[error("The user may have removed the '{0}' service outwith the node manager")] ServiceRemovedManually(String), diff --git a/sn_service_management/src/faucet.rs b/sn_service_management/src/faucet.rs index 7607278bef..fc30f6ad91 100644 --- a/sn_service_management/src/faucet.rs +++ b/sn_service_management/src/faucet.rs @@ -102,7 +102,7 @@ impl<'a> ServiceStateActions for FaucetService<'a> { async fn on_start(&mut self) -> Result<()> { let pid = self .service_control - .get_process_pid(&self.service_data.service_name)?; + .get_process_pid(&self.service_data.faucet_path)?; self.service_data.pid = Some(pid); self.service_data.status = ServiceStatus::Running; Ok(()) diff --git a/sn_service_management/src/lib.rs b/sn_service_management/src/lib.rs index 457e0926bd..9a2dca1f5e 100644 --- a/sn_service_management/src/lib.rs +++ b/sn_service_management/src/lib.rs @@ -49,6 +49,7 @@ pub enum UpgradeResult { Forced(String, String), NotRequired, Upgraded(String, String), + UpgradedButNotStarted(String, String, String), Error(String), } From 3626090efab234a54dd9ad56b1923f9618d362c8 Mon Sep 17 00:00:00 2001 From: qima Date: Thu, 11 Apr 2024 16:24:37 +0800 Subject: [PATCH 055/205] fix(node): fetcher completes on_going_fetch entry on record_key only --- sn_networking/src/event.rs | 7 ++++++- sn_networking/src/record_store.rs | 2 +- sn_networking/src/replication_fetcher.rs | 6 +++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index bc13430e8b..a1ec20a448 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -1307,7 +1307,12 @@ impl SwarmDriver { // if we have a local value of matching record_type, we don't need to fetch it if let Some((_, local_record_type)) = local { - local_record_type != record_type + let not_same_type = local_record_type != record_type; + if not_same_type { + // Shall only happens for Register + info!("Record {addr:?} has different type: local {local_record_type:?}, incoming {record_type:?}"); + } + not_same_type } else { true } diff --git a/sn_networking/src/record_store.rs b/sn_networking/src/record_store.rs index 88eccb2079..9430081f8b 100644 --- a/sn_networking/src/record_store.rs +++ b/sn_networking/src/record_store.rs @@ -582,7 +582,7 @@ impl RecordStore for NodeRecordStore { // ignored if we don't have the record locally. let key = PrettyPrintRecordKey::from(k); if !self.records.contains_key(k) { - trace!("Record not found locally: {key}"); + trace!("Record not found locally: {key:?}"); return None; } diff --git a/sn_networking/src/replication_fetcher.rs b/sn_networking/src/replication_fetcher.rs index a5cdbdafc5..e21f56c635 100644 --- a/sn_networking/src/replication_fetcher.rs +++ b/sn_networking/src/replication_fetcher.rs @@ -134,6 +134,10 @@ impl ReplicationFetcher { // Notify the replication fetcher about a newly added Record to the node. // The corresponding key can now be removed from the replication fetcher. // Also returns the next set of keys that has to be fetched from the peer/network. + // + // Note: for Register, which different content (i.e. record_type) bearing same record_key + // remove `on_going_fetches` entry bearing same `record_key` only, + // to avoid false FetchFailed alarm against the peer. pub(crate) fn notify_about_new_put( &mut self, new_put: RecordKey, @@ -143,7 +147,7 @@ impl ReplicationFetcher { .retain(|(key, t, _), _| key != &new_put || t != &record_type); // if we're actively fetching for the key, reduce the on_going_fetches - let _ = self.on_going_fetches.remove(&(new_put, record_type)); + self.on_going_fetches.retain(|(key, _t), _| key != &new_put); self.next_keys_to_fetch() } From 9d57535244cae65abd5ed30a8473da0b0fb3d412 Mon Sep 17 00:00:00 2001 From: qima Date: Sat, 13 Apr 2024 00:34:35 +0800 Subject: [PATCH 056/205] fix(node): notify replication_fetcher of early completion --- sn_networking/src/cmd.rs | 20 ++++++++++++++++++++ sn_networking/src/lib.rs | 6 ++++++ sn_networking/src/replication_fetcher.rs | 12 ++++++++++++ sn_node/src/put_validation.rs | 5 +++++ 4 files changed, 43 insertions(+) diff --git a/sn_networking/src/cmd.rs b/sn_networking/src/cmd.rs index 744c43f2bc..147603f560 100644 --- a/sn_networking/src/cmd.rs +++ b/sn_networking/src/cmd.rs @@ -177,6 +177,8 @@ pub enum SwarmCmd { QuoteVerification { quotes: Vec<(PeerId, PaymentQuote)>, }, + // Notify a fetch completion + FetchCompleted(RecordKey), } /// Debug impl for SwarmCmd to avoid printing full Record, instead only RecodKey @@ -297,6 +299,13 @@ impl Debug for SwarmCmd { SwarmCmd::QuoteVerification { quotes } => { write!(f, "SwarmCmd::QuoteVerification of {} quotes", quotes.len()) } + SwarmCmd::FetchCompleted(key) => { + write!( + f, + "SwarmCmd::FetchCompleted({:?})", + PrettyPrintRecordKey::from(key) + ) + } } } } @@ -727,6 +736,17 @@ impl SwarmDriver { self.verify_peer_quote(peer_id, quote); } } + SwarmCmd::FetchCompleted(key) => { + info!( + "Fetch {:?} early completed, may fetched an old version record.", + PrettyPrintRecordKey::from(&key) + ); + cmd_string = "FetchCompleted"; + let new_keys_to_fetch = self.replication_fetcher.notify_fetch_early_completed(key); + if !new_keys_to_fetch.is_empty() { + self.send_event(NetworkEvent::KeysToFetchForReplication(new_keys_to_fetch)); + } + } } self.log_handling(cmd_string.to_string(), start.elapsed()); diff --git a/sn_networking/src/lib.rs b/sn_networking/src/lib.rs index 62046b2484..0851efd682 100644 --- a/sn_networking/src/lib.rs +++ b/sn_networking/src/lib.rs @@ -681,6 +681,12 @@ impl Network { response } + /// Notify ReplicationFetch a fetch attempt is completed. + /// (but it won't trigger any real writes to disk, say fetched an old version of register) + pub fn notify_fetch_completed(&self, key: RecordKey) { + self.send_swarm_cmd(SwarmCmd::FetchCompleted(key)) + } + /// Put `Record` to the local RecordStore /// Must be called after the validations are performed on the Record pub fn put_local_record(&self, record: Record) { diff --git a/sn_networking/src/replication_fetcher.rs b/sn_networking/src/replication_fetcher.rs index e21f56c635..c26f1d2845 100644 --- a/sn_networking/src/replication_fetcher.rs +++ b/sn_networking/src/replication_fetcher.rs @@ -152,6 +152,18 @@ impl ReplicationFetcher { self.next_keys_to_fetch() } + // An early completion of a fetch means the target is an old version record (Register or Spend). + pub(crate) fn notify_fetch_early_completed( + &mut self, + key_in: RecordKey, + ) -> Vec<(PeerId, RecordKey)> { + self.to_be_fetched.retain(|(key, _t, _), _| key != &key_in); + + self.on_going_fetches.retain(|(key, _t), _| key != &key_in); + + self.next_keys_to_fetch() + } + // Returns the set of keys that has to be fetched from the peer/network. // Target must not be under-fetching // and no more than MAX_PARALLEL_FETCH fetches to be undertaken at the same time. diff --git a/sn_node/src/put_validation.rs b/sn_node/src/put_validation.rs index d0c07a82b3..9dcc5e459b 100644 --- a/sn_node/src/put_validation.rs +++ b/sn_node/src/put_validation.rs @@ -89,6 +89,9 @@ impl Node { record_key, RecordType::NonChunk(content_hash), ); + } else { + // Notify replication_fetcher to mark the attempt as completed. + self.network.notify_fetch_completed(record.key.clone()); } result } @@ -289,6 +292,8 @@ impl Node { let updated_register = match self.register_validation(®ister, present_locally).await? { Some(reg) => reg, None => { + // Notify replication_fetcher to mark the attempt as completed. + self.network.notify_fetch_completed(key.clone()); return Ok(CmdOk::DataAlreadyPresent); } }; From 5c64387dcf2d2fa558c6d0e1f2b112e8c86a185d Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 3 Apr 2024 19:07:36 +0100 Subject: [PATCH 057/205] docs: clarify client::new description Just removed a few words to try and effect another change in `sn_client`, to get a new version. This crate has already been published and references an older version of `sn_protocol`, which is causing a problem because `sn_cli` is referencing a newer version of it. --- Cargo.lock | 197 ++++++++++++++++++++++++------------------- sn_client/src/api.rs | 3 +- 2 files changed, 112 insertions(+), 88 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8540c35f25..ab30d9b995 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -118,9 +118,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.16" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "android-tzdata" @@ -193,9 +193,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.81" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" +checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" [[package]] name = "arrayref" @@ -344,9 +344,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.79" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", @@ -716,9 +716,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.15.4" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "byte-slice-cast" @@ -794,9 +794,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.90" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" +checksum = "17f6e324229dc011159fcc089755d1e2e216a90d43a7dea6853ca740b84f35e7" dependencies = [ "jobserver", "libc", @@ -843,7 +843,7 @@ dependencies = [ "js-sys", "num-traits", "wasm-bindgen", - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -1491,9 +1491,9 @@ dependencies = [ [[package]] name = "either" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" [[package]] name = "encode_unicode" @@ -1503,9 +1503,9 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "encoding_rs" -version = "0.8.33" +version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ "cfg-if", ] @@ -1865,9 +1865,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" dependencies = [ "cfg-if", "js-sys", @@ -1984,9 +1984,9 @@ dependencies = [ [[package]] name = "half" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eceaaeec696539ddaf7b333340f1af35a5aa87ae3e4f3ead0532f72affab2e" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ "cfg-if", "crunchy", @@ -2565,7 +2565,7 @@ dependencies = [ "socket2", "widestring", "windows-sys 0.48.0", - "winreg", + "winreg 0.50.0", ] [[package]] @@ -2611,9 +2611,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.28" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" +checksum = "685a7d121ee3f65ae4fddd72b25a04bb36b6af81bc0828f7d5434c0fe60fa3a2" dependencies = [ "libc", ] @@ -3530,9 +3530,9 @@ dependencies = [ [[package]] name = "num" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af" +checksum = "3135b08af27d103b0a51f2ae0f8632117b7b185ccf931445affa8df530576a41" dependencies = [ "num-bigint", "num-complex", @@ -3863,11 +3863,11 @@ dependencies = [ [[package]] name = "pem" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310" +checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" dependencies = [ - "base64 0.21.7", + "base64 0.22.0", "serde", ] @@ -4177,9 +4177,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.79" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" +checksum = "a56dea16b0a29e94408b9aa5e2940a4eedbd128a1ba20e8f7ae60fd3d465af0e" dependencies = [ "unicode-ident", ] @@ -4410,9 +4410,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -4596,16 +4596,16 @@ dependencies = [ "wasm-bindgen-futures", "web-sys", "webpki-roots 0.25.4", - "winreg", + "winreg 0.50.0", ] [[package]] name = "reqwest" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d66674f2b6fb864665eea7a3c1ac4e3dfacd2fda83cf6f935a612e01b0e3338" +checksum = "3e6cc1e89e689536eb5aeede61520e874df5a4707df811cd5da4aa5fbb2aae19" dependencies = [ - "base64 0.21.7", + "base64 0.22.0", "bytes", "futures-core", "futures-util", @@ -4623,7 +4623,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rustls 0.22.3", - "rustls-pemfile 1.0.4", + "rustls-pemfile 2.1.2", "rustls-pki-types", "serde", "serde_json", @@ -4637,7 +4637,7 @@ dependencies = [ "wasm-bindgen-futures", "web-sys", "webpki-roots 0.26.1", - "winreg", + "winreg 0.52.0", ] [[package]] @@ -4838,6 +4838,16 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pemfile" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +dependencies = [ + "base64 0.22.0", + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.4.1" @@ -4867,9 +4877,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" [[package]] name = "rusty-fork" @@ -5184,7 +5194,7 @@ dependencies = [ "nix 0.27.1", "predicates 3.1.0", "prost 0.9.0", - "reqwest 0.12.2", + "reqwest 0.12.3", "semver", "serde", "serde_json", @@ -5272,7 +5282,7 @@ dependencies = [ "libp2p", "rand", "rayon", - "reqwest 0.12.2", + "reqwest 0.12.3", "rmp-serde", "serde", "sn_build_info", @@ -5471,7 +5481,7 @@ dependencies = [ "prost 0.9.0", "rand", "rayon", - "reqwest 0.12.2", + "reqwest 0.12.3", "rmp-serde", "self_encryption", "serde", @@ -5535,7 +5545,7 @@ dependencies = [ "lazy_static", "libp2p", "rand", - "reqwest 0.12.2", + "reqwest 0.12.3", "sn_protocol", "thiserror", "tokio", @@ -5839,9 +5849,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.30.8" +version = "0.30.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b1a378e48fb3ce3a5cf04359c456c9c98ff689bcf1c1bc6e6a31f247686f275" +checksum = "26d7c217777061d5a2d652aea771fb9ba98b6dade657204b08c4b9604d11555b" dependencies = [ "cfg-if", "core-foundation-sys", @@ -5960,9 +5970,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.34" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", @@ -5983,9 +5993,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ "num-conv", "time-core", @@ -6137,9 +6147,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.20.1" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" +checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" dependencies = [ "futures-util", "log", @@ -6455,14 +6465,14 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" -version = "0.20.1" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" +checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" dependencies = [ "byteorder", "bytes", "data-encoding", - "http 0.2.12", + "http 1.1.0", "httparse", "log", "rand", @@ -6698,9 +6708,9 @@ dependencies = [ [[package]] name = "warp" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e92e22e03ff1230c03a1a8ee37d2f89cd489e2e541b7550d6afad96faed169" +checksum = "4378d202ff965b011c64817db11d5829506d3404edeadb61f190d111da3f231c" dependencies = [ "bytes", "futures-channel", @@ -6714,13 +6724,11 @@ dependencies = [ "multer", "percent-encoding", "pin-project", - "rustls-pemfile 1.0.4", "scoped-tls", "serde", "serde_json", "serde_urlencoded", "tokio", - "tokio-stream", "tokio-tungstenite", "tokio-util 0.7.10", "tower-service", @@ -6872,9 +6880,9 @@ dependencies = [ [[package]] name = "widestring" -version = "1.0.2" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "653f141f39ec16bba3c5abe400a0c60da7468261cc2cbf36805022876bc721a8" +checksum = "7219d36b6eac893fa81e84ebe06485e7dcbb616177469b142df14f1f4deb1311" [[package]] name = "winapi" @@ -6924,7 +6932,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ "windows-core 0.52.0", - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -6942,7 +6950,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -6960,7 +6968,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -6980,17 +6988,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.52.4", - "windows_aarch64_msvc 0.52.4", - "windows_i686_gnu 0.52.4", - "windows_i686_msvc 0.52.4", - "windows_x86_64_gnu 0.52.4", - "windows_x86_64_gnullvm 0.52.4", - "windows_x86_64_msvc 0.52.4", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] [[package]] @@ -7001,9 +7010,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" [[package]] name = "windows_aarch64_msvc" @@ -7013,9 +7022,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" [[package]] name = "windows_i686_gnu" @@ -7025,9 +7034,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" [[package]] name = "windows_i686_msvc" @@ -7037,9 +7052,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" [[package]] name = "windows_x86_64_gnu" @@ -7049,9 +7064,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" [[package]] name = "windows_x86_64_gnullvm" @@ -7061,9 +7076,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] name = "windows_x86_64_msvc" @@ -7073,9 +7088,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "winreg" @@ -7087,6 +7102,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "winreg" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "wyz" version = "0.5.1" diff --git a/sn_client/src/api.rs b/sn_client/src/api.rs index 626c3c389f..4fdd4d0cd9 100644 --- a/sn_client/src/api.rs +++ b/sn_client/src/api.rs @@ -64,8 +64,7 @@ impl Client { /// Instantiate a new client. /// - /// Optionally specify the duration for the connection timeout. - /// + /// Optionally specify the maximum time the client will wait for a connection before timing out. /// Defaults to 180 seconds. /// /// # Arguments From c6a5f6969c19db21f38461765e5cbd91668f9a7e Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 3 Apr 2024 23:22:06 +0100 Subject: [PATCH 058/205] docs: clarify client documentation Again, these changes are just to effect a release. A change to the release workflow also removes testing the environment variable and re-exporting it. On the last build I ran, it appears to have worked as expected, where previously it didn't. Doesn't really make any sense. --- sn_client/src/api.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sn_client/src/api.rs b/sn_client/src/api.rs index 4fdd4d0cd9..626c3c389f 100644 --- a/sn_client/src/api.rs +++ b/sn_client/src/api.rs @@ -64,7 +64,8 @@ impl Client { /// Instantiate a new client. /// - /// Optionally specify the maximum time the client will wait for a connection before timing out. + /// Optionally specify the duration for the connection timeout. + /// /// Defaults to 180 seconds. /// /// # Arguments From ee4085b39e641206546aadd5a523b61d724e35f7 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Wed, 10 Apr 2024 10:52:20 +0900 Subject: [PATCH 059/205] chore(release): sn_auditor-v0.1.7/sn_client-v0.105.3/sn_networking-v0.14.4/sn_protocol-v0.16.3/sn_build_info-v0.1.7/sn_transfers-v0.17.2/sn_peers_acquisition-v0.2.10/sn_cli-v0.90.4/sn_faucet-v0.4.9/sn_metrics-v0.1.4/sn_node-v0.105.6/sn_service_management-v0.2.4/sn-node-manager-v0.7.4/sn_node_rpc_client-v0.6.8/token_supplies-v0.1.47 --- Cargo.lock | 30 +++++++++++++++--------------- sn_auditor/Cargo.toml | 6 +++--- sn_build_info/Cargo.toml | 2 +- sn_cli/Cargo.toml | 12 ++++++------ sn_client/Cargo.toml | 12 ++++++------ sn_faucet/Cargo.toml | 10 +++++----- sn_metrics/Cargo.toml | 2 +- sn_networking/Cargo.toml | 6 +++--- sn_node/Cargo.toml | 18 +++++++++--------- sn_node_manager/Cargo.toml | 10 +++++----- sn_node_rpc_client/Cargo.toml | 14 +++++++------- sn_peers_acquisition/Cargo.toml | 4 ++-- sn_protocol/Cargo.toml | 6 +++--- sn_service_management/Cargo.toml | 4 ++-- sn_transfers/Cargo.toml | 2 +- token_supplies/Cargo.toml | 2 +- 16 files changed, 70 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ab30d9b995..d4fd66573e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5177,7 +5177,7 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "sn-node-manager" -version = "0.7.2" +version = "0.7.4" dependencies = [ "assert_cmd", "assert_fs", @@ -5234,7 +5234,7 @@ dependencies = [ [[package]] name = "sn_auditor" -version = "0.1.2" +version = "0.1.7" dependencies = [ "blsttc", "clap", @@ -5253,14 +5253,14 @@ dependencies = [ [[package]] name = "sn_build_info" -version = "0.1.6" +version = "0.1.7" dependencies = [ "vergen", ] [[package]] name = "sn_cli" -version = "0.90.2" +version = "0.90.4" dependencies = [ "aes 0.7.5", "base64 0.22.0", @@ -5301,7 +5301,7 @@ dependencies = [ [[package]] name = "sn_client" -version = "0.105.2" +version = "0.105.3" dependencies = [ "assert_matches", "async-trait", @@ -5350,7 +5350,7 @@ dependencies = [ [[package]] name = "sn_faucet" -version = "0.4.3" +version = "0.4.9" dependencies = [ "assert_fs", "base64 0.22.0", @@ -5403,7 +5403,7 @@ dependencies = [ [[package]] name = "sn_metrics" -version = "0.1.3" +version = "0.1.4" dependencies = [ "clap", "color-eyre", @@ -5417,7 +5417,7 @@ dependencies = [ [[package]] name = "sn_networking" -version = "0.14.1" +version = "0.14.4" dependencies = [ "aes-gcm-siv", "async-trait", @@ -5457,7 +5457,7 @@ dependencies = [ [[package]] name = "sn_node" -version = "0.105.3" +version = "0.105.6" dependencies = [ "assert_fs", "assert_matches", @@ -5512,7 +5512,7 @@ dependencies = [ [[package]] name = "sn_node_rpc_client" -version = "0.6.3" +version = "0.6.8" dependencies = [ "assert_fs", "async-trait", @@ -5539,7 +5539,7 @@ dependencies = [ [[package]] name = "sn_peers_acquisition" -version = "0.2.8" +version = "0.2.10" dependencies = [ "clap", "lazy_static", @@ -5555,7 +5555,7 @@ dependencies = [ [[package]] name = "sn_protocol" -version = "0.16.1" +version = "0.16.3" dependencies = [ "blsttc", "bytes", @@ -5601,7 +5601,7 @@ dependencies = [ [[package]] name = "sn_service_management" -version = "0.2.1" +version = "0.2.4" dependencies = [ "async-trait", "dirs-next", @@ -5625,7 +5625,7 @@ dependencies = [ [[package]] name = "sn_transfers" -version = "0.17.1" +version = "0.17.2" dependencies = [ "assert_fs", "blsttc", @@ -6052,7 +6052,7 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "token_supplies" -version = "0.1.46" +version = "0.1.47" dependencies = [ "dirs-next", "reqwest 0.11.27", diff --git a/sn_auditor/Cargo.toml b/sn_auditor/Cargo.toml index c085639f5f..dfa7c9e3a0 100644 --- a/sn_auditor/Cargo.toml +++ b/sn_auditor/Cargo.toml @@ -2,7 +2,7 @@ authors = ["MaidSafe Developers "] description = "Safe Network Auditor" name = "sn_auditor" -version = "0.1.2" +version = "0.1.7" edition = "2021" homepage = "https://maidsafe.net" repository = "https://github.com/maidsafe/safe_network" @@ -24,9 +24,9 @@ dirs-next = "~2.0.0" graphviz-rust = "0.9.0" serde = { version = "1.0.133", features = [ "derive", "rc" ]} serde_json = "1.0.108" -sn_client = { path = "../sn_client", version = "0.105.2" } +sn_client = { path = "../sn_client", version = "0.105.3" } sn_logging = { path = "../sn_logging", version = "0.2.25" } -sn_peers_acquisition= { path="../sn_peers_acquisition", version = "0.2.8" } +sn_peers_acquisition= { path="../sn_peers_acquisition", version = "0.2.10" } tiny_http = { version="0.12", features = ["ssl-rustls"] } tracing = { version = "~0.1.26" } tokio = { version = "1.32.0", features = ["io-util", "macros", "parking_lot", "rt", "sync", "time", "fs"] } diff --git a/sn_build_info/Cargo.toml b/sn_build_info/Cargo.toml index 49441962c3..6fbd8fe1f8 100644 --- a/sn_build_info/Cargo.toml +++ b/sn_build_info/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_build_info" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.1.6" +version = "0.1.7" [build-dependencies] vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] } diff --git a/sn_cli/Cargo.toml b/sn_cli/Cargo.toml index ea48f7655d..d0cf2e521d 100644 --- a/sn_cli/Cargo.toml +++ b/sn_cli/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_cli" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.90.2" +version = "0.90.4" [[bin]] path="src/bin/main.rs" @@ -52,11 +52,11 @@ rayon = "1.8.0" reqwest = { version="0.12.2", default-features=false, features = ["rustls-tls-manual-roots"] } rmp-serde = "1.1.1" serde = { version = "1.0.133", features = [ "derive"]} -sn_build_info = { path="../sn_build_info", version = "0.1.5" } -sn_client = { path = "../sn_client", version = "0.105.2" } +sn_build_info = { path="../sn_build_info", version = "0.1.7" } +sn_client = { path = "../sn_client", version = "0.105.3" } sn_logging = { path = "../sn_logging", version = "0.2.25" } -sn_peers_acquisition= { path="../sn_peers_acquisition", version = "0.2.8" } -sn_protocol = { path = "../sn_protocol", version = "0.16.1" } +sn_peers_acquisition= { path="../sn_peers_acquisition", version = "0.2.10" } +sn_protocol = { path = "../sn_protocol", version = "0.16.3" } tempfile = "3.6.0" tiny-keccak = "~2.0.2" tokio = { version = "1.32.0", features = ["io-util", "macros", "parking_lot", "rt", "sync", "time", "fs"] } @@ -70,7 +70,7 @@ eyre = "0.6.8" criterion = "0.5.1" tempfile = "3.6.0" rand = { version = "~0.8.5", features = ["small_rng"] } -sn_client = { path = "../sn_client", version = "0.105.2", features = ["test-utils"] } +sn_client = { path = "../sn_client", version = "0.105.3", features = ["test-utils"] } [lints] workspace = true diff --git a/sn_client/Cargo.toml b/sn_client/Cargo.toml index 9fbf5532a8..3bd0c2e36a 100644 --- a/sn_client/Cargo.toml +++ b/sn_client/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_client" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.105.2" +version = "0.105.3" [features] default=[] @@ -36,18 +36,18 @@ rayon = "1.8.0" rmp-serde = "1.1.1" self_encryption = "~0.29.0" serde = { version = "1.0.133", features = [ "derive", "rc" ]} -sn_networking = { path = "../sn_networking", version = "0.14.1" } -sn_protocol = { path = "../sn_protocol", version = "0.16.1" } +sn_networking = { path = "../sn_networking", version = "0.14.4" } +sn_protocol = { path = "../sn_protocol", version = "0.16.3" } serde_json = "1.0" sn_registers = { path = "../sn_registers", version = "0.3.12" } -sn_transfers = { path = "../sn_transfers", version = "0.17.1" } +sn_transfers = { path = "../sn_transfers", version = "0.17.2" } tempfile = "3.6.0" thiserror = "1.0.23" tiny-keccak = "~2.0.2" tokio = { version = "1.35.0", features = ["io-util", "macros", "rt", "sync", "time"] } tracing = { version = "~0.1.26" } xor_name = "5.0.0" -sn_peers_acquisition= { path="../sn_peers_acquisition", version = "0.2.8", optional = true } +sn_peers_acquisition= { path="../sn_peers_acquisition", version = "0.2.10", optional = true } eyre = { version = "0.6.8", optional = true } lazy_static = { version = "~1.4.0", optional = true } @@ -71,7 +71,7 @@ crate-type = ["cdylib", "rlib"] getrandom = { version = "0.2.12", features = ["js"] } wasm-bindgen = "0.2.90" wasm-bindgen-futures = "0.4.40" -sn_peers_acquisition= { path="../sn_peers_acquisition", version = "0.2.8" } +sn_peers_acquisition= { path="../sn_peers_acquisition", version = "0.2.10" } console_error_panic_hook = "0.1.6" tracing-wasm = "0.2.1" wasmtimer = "0.2.0" diff --git a/sn_faucet/Cargo.toml b/sn_faucet/Cargo.toml index cdab9a8378..40b4c218ee 100644 --- a/sn_faucet/Cargo.toml +++ b/sn_faucet/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_faucet" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.4.3" +version = "0.4.9" [features] default = [] @@ -35,11 +35,11 @@ indicatif = { version = "0.17.5", features = ["tokio"] } minreq = { version = "2.11.0", features = ["https-rustls"], optional = true } serde = { version = "1.0.193", features = ["derive"] } serde_json = "1.0.108" -sn_build_info = { path = "../sn_build_info", version = "0.1.5" } -sn_client = { path = "../sn_client", version = "0.105.2" } +sn_build_info = { path = "../sn_build_info", version = "0.1.7" } +sn_client = { path = "../sn_client", version = "0.105.3" } sn_logging = { path = "../sn_logging", version = "0.2.25" } -sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.8" } -sn_transfers = { path = "../sn_transfers", version = "0.17.1" } +sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.10" } +sn_transfers = { path = "../sn_transfers", version = "0.17.2" } tokio = { version = "1.32.0", features = ["parking_lot", "rt"] } tracing = { version = "~0.1.26" } url = "2.5.0" diff --git a/sn_metrics/Cargo.toml b/sn_metrics/Cargo.toml index 2fe4db2a56..4eaed01086 100644 --- a/sn_metrics/Cargo.toml +++ b/sn_metrics/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_metrics" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.1.3" +version = "0.1.4" [[bin]] path = "src/main.rs" diff --git a/sn_networking/Cargo.toml b/sn_networking/Cargo.toml index ef985f527c..b1a019e79f 100644 --- a/sn_networking/Cargo.toml +++ b/sn_networking/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_networking" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.14.1" +version = "0.14.4" [features] default = ["libp2p/quic"] @@ -49,8 +49,8 @@ rand = { version = "~0.8.5", features = ["small_rng"] } rayon = "1.8.0" rmp-serde = "1.1.1" serde = { version = "1.0.133", features = ["derive", "rc"] } -sn_protocol = { path = "../sn_protocol", version = "0.16.1" } -sn_transfers = { path = "../sn_transfers", version = "0.17.1" } +sn_protocol = { path = "../sn_protocol", version = "0.16.3" } +sn_transfers = { path = "../sn_transfers", version = "0.17.2" } sn_registers = { path = "../sn_registers", version = "0.3.12" } sysinfo = { version = "0.30.8", default-features = false, optional = true } thiserror = "1.0.23" diff --git a/sn_node/Cargo.toml b/sn_node/Cargo.toml index 89356652f3..36ac08adef 100644 --- a/sn_node/Cargo.toml +++ b/sn_node/Cargo.toml @@ -2,7 +2,7 @@ authors = ["MaidSafe Developers "] description = "Safe Node" name = "sn_node" -version = "0.105.3" +version = "0.105.6" edition = "2021" license = "GPL-3.0" homepage = "https://maidsafe.net" @@ -49,15 +49,15 @@ rmp-serde = "1.1.1" rayon = "1.8.0" self_encryption = "~0.29.0" serde = { version = "1.0.133", features = ["derive", "rc"] } -sn_build_info = { path = "../sn_build_info", version = "0.1.5" } -sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.8" } -sn_client = { path = "../sn_client", version = "0.105.2" } +sn_build_info = { path = "../sn_build_info", version = "0.1.7" } +sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.10" } +sn_client = { path = "../sn_client", version = "0.105.3" } sn_logging = { path = "../sn_logging", version = "0.2.25" } -sn_networking = { path = "../sn_networking", version = "0.14.1" } -sn_protocol = { path = "../sn_protocol", version = "0.16.1" } +sn_networking = { path = "../sn_networking", version = "0.14.4" } +sn_protocol = { path = "../sn_protocol", version = "0.16.3" } sn_registers = { path = "../sn_registers", version = "0.3.12" } -sn_transfers = { path = "../sn_transfers", version = "0.17.1" } -sn_service_management = { path = "../sn_service_management", version = "0.2.1" } +sn_transfers = { path = "../sn_transfers", version = "0.17.2" } +sn_service_management = { path = "../sn_service_management", version = "0.2.4" } thiserror = "1.0.23" tokio = { version = "1.32.0", features = [ "io-util", @@ -84,7 +84,7 @@ reqwest = { version = "0.12.2", default-features = false, features = [ "rustls-tls-manual-roots", ] } serde_json = "1.0" -sn_protocol = { path = "../sn_protocol", version = "0.16.1", features = [ +sn_protocol = { path = "../sn_protocol", version = "0.16.3", features = [ "rpc", ] } tempfile = "3.6.0" diff --git a/sn_node_manager/Cargo.toml b/sn_node_manager/Cargo.toml index 8c51e501fa..ffdfa4f97b 100644 --- a/sn_node_manager/Cargo.toml +++ b/sn_node_manager/Cargo.toml @@ -7,7 +7,7 @@ license = "GPL-3.0" name = "sn-node-manager" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.7.2" +version = "0.7.4" [[bin]] name = "safenode-manager" @@ -41,11 +41,11 @@ semver = "1.0.20" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" service-manager = "0.6.0" -sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.8" } -sn_protocol = { path = "../sn_protocol", version = "0.16.1" } -sn_service_management = { path = "../sn_service_management", version = "0.2.1" } +sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.10" } +sn_protocol = { path = "../sn_protocol", version = "0.16.3" } +sn_service_management = { path = "../sn_service_management", version = "0.2.4" } sn-releases = "0.2.0" -sn_transfers = { path = "../sn_transfers", version = "0.17.1" } +sn_transfers = { path = "../sn_transfers", version = "0.17.2" } sysinfo = "0.30.8" tokio = { version = "1.26", features = ["full"] } tracing = { version = "~0.1.26" } diff --git a/sn_node_rpc_client/Cargo.toml b/sn_node_rpc_client/Cargo.toml index 74837b90a8..0a2e096ac4 100644 --- a/sn_node_rpc_client/Cargo.toml +++ b/sn_node_rpc_client/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_node_rpc_client" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.6.3" +version = "0.6.8" [[bin]] name = "safenode_rpc_client" @@ -23,13 +23,13 @@ color-eyre = "0.6.2" hex = "~0.4.3" libp2p = { version="0.53", features = ["kad"]} libp2p-identity = { version="0.2.7", features = ["rand"] } -sn_client = { path = "../sn_client", version = "0.105.2" } +sn_client = { path = "../sn_client", version = "0.105.3" } sn_logging = { path = "../sn_logging", version = "0.2.25" } -sn_node = { path = "../sn_node", version = "0.105.3" } -sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.8" } -sn_protocol = { path = "../sn_protocol", version = "0.16.1", features=["rpc"] } -sn_service_management = { path = "../sn_service_management", version = "0.2.1" } -sn_transfers = { path = "../sn_transfers", version = "0.17.1" } +sn_node = { path = "../sn_node", version = "0.105.6" } +sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.10" } +sn_protocol = { path = "../sn_protocol", version = "0.16.3", features=["rpc"] } +sn_service_management = { path = "../sn_service_management", version = "0.2.4" } +sn_transfers = { path = "../sn_transfers", version = "0.17.2" } thiserror = "1.0.23" # # watch out updating this, protoc compiler needs to be installed on all build systems # # arm builds + musl are very problematic diff --git a/sn_peers_acquisition/Cargo.toml b/sn_peers_acquisition/Cargo.toml index a59695b7c0..ed13114940 100644 --- a/sn_peers_acquisition/Cargo.toml +++ b/sn_peers_acquisition/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_peers_acquisition" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.2.8" +version = "0.2.10" [features] local-discovery = [] @@ -21,7 +21,7 @@ lazy_static = "~1.4.0" libp2p = { version="0.53", features = [] } rand = "0.8.5" reqwest = { version="0.12.2", default-features=false, features = ["rustls-tls"], optional = true } -sn_protocol = { path = "../sn_protocol", version = "0.16.1" } +sn_protocol = { path = "../sn_protocol", version = "0.16.3" } thiserror = "1.0.23" tokio = { version = "1.32.0", optional = true, default-features = false} tracing = { version = "~0.1.26" } diff --git a/sn_protocol/Cargo.toml b/sn_protocol/Cargo.toml index f2aa7a1cf4..8d5e36307e 100644 --- a/sn_protocol/Cargo.toml +++ b/sn_protocol/Cargo.toml @@ -7,7 +7,7 @@ license = "GPL-3.0" name = "sn_protocol" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.16.1" +version = "0.16.3" [features] default = [] @@ -28,8 +28,8 @@ rmp-serde = "1.1.1" serde = { version = "1.0.133", features = [ "derive", "rc" ]} serde_json = "1.0" sha2 = "0.10.7" -sn_build_info = { path="../sn_build_info", version = "0.1.5" } -sn_transfers = { path = "../sn_transfers", version = "0.17.1" } +sn_build_info = { path="../sn_build_info", version = "0.1.7" } +sn_transfers = { path = "../sn_transfers", version = "0.17.2" } sn_registers = { path = "../sn_registers", version = "0.3.12" } thiserror = "1.0.23" tiny-keccak = { version = "~2.0.2", features = [ "sha3" ] } diff --git a/sn_service_management/Cargo.toml b/sn_service_management/Cargo.toml index aea1beea99..d8c6d274c8 100644 --- a/sn_service_management/Cargo.toml +++ b/sn_service_management/Cargo.toml @@ -7,7 +7,7 @@ license = "GPL-3.0" name = "sn_service_management" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.2.1" +version = "0.2.4" [dependencies] async-trait = "0.1" @@ -19,7 +19,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" semver = "1.0.20" service-manager = "0.6.0" -sn_protocol = { path = "../sn_protocol", version = "0.16.1", features = ["rpc"] } +sn_protocol = { path = "../sn_protocol", version = "0.16.3", features = ["rpc"] } sysinfo = "0.30.8" thiserror = "1.0.23" tokio = { version = "1.32.0", features = ["time"] } diff --git a/sn_transfers/Cargo.toml b/sn_transfers/Cargo.toml index 483b24acd2..be8a1ba351 100644 --- a/sn_transfers/Cargo.toml +++ b/sn_transfers/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_transfers" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.17.1" +version = "0.17.2" [dependencies] bls = { package = "blsttc", version = "8.0.1" } diff --git a/token_supplies/Cargo.toml b/token_supplies/Cargo.toml index 8e50a02549..86d58055a6 100644 --- a/token_supplies/Cargo.toml +++ b/token_supplies/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "token_supplies" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.1.46" +version = "0.1.47" [dependencies] From 8e4f66c61154863b1111185e48f4c93828016f88 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 11 Apr 2024 14:29:03 +0900 Subject: [PATCH 060/205] ci: restrict stable releases to stable branch --- .github/workflows/version_bump.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml index 185a91834e..2af4b2ed46 100644 --- a/.github/workflows/version_bump.yml +++ b/.github/workflows/version_bump.yml @@ -63,7 +63,12 @@ jobs: sudo mv release-plz /usr/local/bin - shell: bash # run as the branch name with the suffix from workflow dispatch, allowing for an empty suffix as a valid option - run: ./resources/scripts/bump_version.sh ${{ github.event.inputs.suffix }} + run: | + if [[ -z "${{ github.event.inputs.suffix }}" && "${{ github.ref_name }}" != "stable" ]]; then + echo "Error: Empty suffix is only allowed on the stable branch." + exit 1 + fi + ./resources/scripts/bump_version.sh ${{ github.event.inputs.suffix }} - name: push version bump commit uses: ad-m/github-push-action@master with: From 5e56d8639bcbc5b704880d1b9c5fd87974d56ed0 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Mon, 15 Apr 2024 14:46:02 +0900 Subject: [PATCH 061/205] ci: dont release on the main branch --- .github/workflows/release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f664b825d9..f23948eba2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,6 @@ concurrency: on: push: branches: - - main - stable - alpha* - beta* @@ -55,7 +54,7 @@ jobs: if [[ "$GITHUB_REF_NAME" != "main" && "$GITHUB_REF_NAME" != "stable" ]]; then echo "NETWORK_VERSION_MODE=restricted" >> $GITHUB_ENV fi - + - name: build release artifacts shell: bash run: | From 8eb22d8fbcfadb643552e3460d8bbcc4e451012d Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Mon, 15 Apr 2024 14:58:25 +0900 Subject: [PATCH 062/205] ci: allow unrestricted alpha/beta builds if on strict branch name --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f23948eba2..78ede9631f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,7 +51,7 @@ jobs: - name: provide network versioning shell: bash run: | - if [[ "$GITHUB_REF_NAME" != "main" && "$GITHUB_REF_NAME" != "stable" ]]; then + if [[ "$GITHUB_REF_NAME" != "main" && "$GITHUB_REF_NAME" != "stable" && "$GITHUB_REF_NAME" != "alpha" && "$GITHUB_REF_NAME" != "beta" ]]; then echo "NETWORK_VERSION_MODE=restricted" >> $GITHUB_ENV fi From 59cec7b7e0167557298a5cde0a2bb1138ae408d2 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Mon, 15 Apr 2024 15:19:02 +0900 Subject: [PATCH 063/205] fix: justfile conditional error --- Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 49bd390fe8..23054ee34f 100644 --- a/Justfile +++ b/Justfile @@ -105,7 +105,7 @@ build-release-artifacts arch: mkdir artifacts cargo clean - if [[ -n "${NETWORK_VERSION_MODE+x}"]]; then + if [[ -n "${NETWORK_VERSION_MODE+x}" ]]; then echo "The NETWORK_VERSION_MODE variable is set to $NETWORK_VERSION_MODE" export CROSS_CONTAINER_OPTS="--env NETWORK_VERSION_MODE=$NETWORK_VERSION_MODE" fi From cb4e0a36b0bd75133d89ab26d7062c18a55c617b Mon Sep 17 00:00:00 2001 From: qima Date: Mon, 15 Apr 2024 19:43:09 +0800 Subject: [PATCH 064/205] test(CI): confirm there is no failed replication fetch --- .github/workflows/merge.yml | 10 ++++++++++ sn_networking/src/replication_fetcher.rs | 4 +++- sn_node/src/node.rs | 3 +++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 3e6de96b1e..820c506688 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -878,6 +878,16 @@ jobs: node_count=$(ls "${{ matrix.node_data_path }}" | wc -l) echo "Node dir count is $node_count" + - name: check there is no failed replication fetch + if: matrix.os != 'windows-latest' # causes error + shell: bash + run: | + if grep -r "failed to fetch" "${{ matrix.node_data_path }}" + then + echo "We find failed replication fetch" + exit 1 + fi + # Only error out after uploading the logs - name: Don't log raw data if: matrix.os != 'windows-latest' # causes error diff --git a/sn_networking/src/replication_fetcher.rs b/sn_networking/src/replication_fetcher.rs index c26f1d2845..1f7046668a 100644 --- a/sn_networking/src/replication_fetcher.rs +++ b/sn_networking/src/replication_fetcher.rs @@ -113,8 +113,10 @@ impl ReplicationFetcher { if !out_of_range_keys.is_empty() { info!("Among {total_incoming_keys} incoming replications from {holder:?}, found {} out of range", out_of_range_keys.len()); + let self_address = NetworkAddress::from_peer(self.self_peer_id); for addr in out_of_range_keys.iter() { - trace!("The incoming record_key {addr:?} is out of range, do not fetch it from {holder:?}"); + let ilog2_distance = self_address.distance(addr).ilog2(); + trace!("The incoming record_key {addr:?} is out of range with ilog2_distance being {ilog2_distance:?}, do not fetch it from {holder:?}"); } } diff --git a/sn_node/src/node.rs b/sn_node/src/node.rs index 85c21942f4..9abcc2aba8 100644 --- a/sn_node/src/node.rs +++ b/sn_node/src/node.rs @@ -411,6 +411,9 @@ impl Node { NetworkEvent::FailedToFetchHolders(bad_nodes) => { event_header = "FailedToFetchHolders"; let network = self.network.clone(); + // Note: this log will be checked in CI, and expecting `not appear`. + // any change to the keyword `failed to fetch` shall incur + // correspondent CI script change as well. error!("Received notification from replication_fetcher, notifying {bad_nodes:?} failed to fetch replication copies from."); let _handle = spawn(async move { for peer_id in bad_nodes { From cccdfc80635b7f97734df4305c69ca88b3ab5669 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Mon, 15 Apr 2024 14:06:27 +0530 Subject: [PATCH 065/205] fix(ci): update the custom branch variable names --- .github/workflows/nightly_wan.yml | 52 ++++++++++++------------- .github/workflows/nightly_wan_churn.yml | 12 +++--- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/.github/workflows/nightly_wan.yml b/.github/workflows/nightly_wan.yml index 6114eb2fb3..afe5f03582 100644 --- a/.github/workflows/nightly_wan.yml +++ b/.github/workflows/nightly_wan.yml @@ -48,12 +48,12 @@ jobs: provider: digital-ocean testnet-name: NightlyE2E # if we were to run on a PR, use the below - # custom-node-bin-org-name: ${{ github.actor }}" - # custom-node-bin-branch-name: ${{ github.event.pull_request.head.ref }} + # safe-network-user: ${{ github.actor }}" + # safe-network-branch: ${{ github.event.pull_request.head.ref }} # Specify custom branch to prevent the deployer from fetching the latest release. # The latest release contains the `network-contacts` feature turned on. - custom-node-bin-org-name: maidsafe - custom-node-bin-branch-name: main + safe-network-user: maidsafe + safe-network-branch: main - name: Check env variables shell: bash @@ -113,8 +113,8 @@ jobs: vm-count: 1 provider: digital-ocean testnet-name: NightlyE2E - custom-node-bin-org-name: maidsafe - custom-node-bin-branch-name: main + safe-network-user: maidsafe + safe-network-branch: main - name: Upload local logs if: always() @@ -143,8 +143,8 @@ jobs: vm-count: 1 provider: digital-ocean testnet-name: NightlyE2E - custom-node-bin-org-name: maidsafe - custom-node-bin-branch-name: main + safe-network-user: maidsafe + safe-network-branch: main # - name: post notification to slack on failure # if: ${{ failure() }} @@ -191,8 +191,8 @@ jobs: vm-count: 1 provider: digital-ocean testnet-name: NightlySpendTest - custom-node-bin-org-name: maidsafe - custom-node-bin-branch-name: main + safe-network-user: maidsafe + safe-network-branch: main - name: Check env variables shell: bash @@ -233,8 +233,8 @@ jobs: vm-count: 1 provider: digital-ocean testnet-name: NightlySpendTest - custom-node-bin-org-name: maidsafe - custom-node-bin-branch-name: main + safe-network-user: maidsafe + safe-network-branch: main - name: Upload local logs if: always() @@ -263,8 +263,8 @@ jobs: vm-count: 1 provider: digital-ocean testnet-name: NightlySpendTest - custom-node-bin-org-name: maidsafe - custom-node-bin-branch-name: main + safe-network-user: maidsafe + safe-network-branch: main # - name: post notification to slack on failure # if: ${{ failure() }} @@ -323,8 +323,8 @@ jobs: vm-count: 1 provider: digital-ocean testnet-name: NightlyChurnTest - custom-node-bin-org-name: maidsafe - custom-node-bin-branch-name: main + safe-network-user: maidsafe + safe-network-branch: main - name: Check env variables shell: bash @@ -361,8 +361,8 @@ jobs: vm-count: 1 provider: digital-ocean testnet-name: NightlyChurnTest - custom-node-bin-org-name: maidsafe - custom-node-bin-branch-name: main + safe-network-user: maidsafe + safe-network-branch: main - name: Upload local logs if: always() @@ -391,8 +391,8 @@ jobs: vm-count: 1 provider: digital-ocean testnet-name: NightlyChurnTest - custom-node-bin-org-name: maidsafe - custom-node-bin-branch-name: main + safe-network-user: maidsafe + safe-network-branch: main # TODO: re-enable the below scripts once we have proper way to restart nodes. # Currently on remote network (not local), the nodes do not handle restart RPC cmd well. They reuse the same @@ -528,8 +528,8 @@ jobs: vm-count: 1 provider: digital-ocean testnet-name: NightlyDataLocationTest - custom-node-bin-org-name: maidsafe - custom-node-bin-branch-name: main + safe-network-user: maidsafe + safe-network-branch: main - name: Check env variables shell: bash @@ -568,8 +568,8 @@ jobs: vm-count: 1 provider: digital-ocean testnet-name: NightlyDataLocationTest - custom-node-bin-org-name: maidsafe - custom-node-bin-branch-name: main + safe-network-user: maidsafe + safe-network-branch: main - name: Upload local logs if: always() @@ -598,8 +598,8 @@ jobs: vm-count: 1 provider: digital-ocean testnet-name: NightlyDataLocationTest - custom-node-bin-org-name: maidsafe - custom-node-bin-branch-name: main + safe-network-user: maidsafe + safe-network-branch: main # TODO: re-enable the below scripts once we have proper way to restart nodes. # Currently on remote network (not local), the nodes do not handle restart RPC cmd well. They reuse the same diff --git a/.github/workflows/nightly_wan_churn.yml b/.github/workflows/nightly_wan_churn.yml index aaaf91755d..4d51eae157 100644 --- a/.github/workflows/nightly_wan_churn.yml +++ b/.github/workflows/nightly_wan_churn.yml @@ -48,8 +48,8 @@ jobs: vm-count: 5 provider: digital-ocean testnet-name: NightlyChurnE2E - custom-node-bin-org-name: maidsafe - custom-node-bin-branch-name: main + safe-network-user: maidsafe + safe-network-branch: main - name: Download material, 1.6G shell: bash @@ -117,8 +117,8 @@ jobs: vm-count: 1 provider: digital-ocean testnet-name: NightlyChurnE2E - custom-node-bin-org-name: maidsafe - custom-node-bin-branch-name: main + safe-network-user: maidsafe + safe-network-branch: main - name: Upload local logs if: always() @@ -147,5 +147,5 @@ jobs: vm-count: 1 provider: digital-ocean testnet-name: NightlyChurnE2E - custom-node-bin-org-name: maidsafe - custom-node-bin-branch-name: main + safe-network-user: maidsafe + safe-network-branch: main From 07dc9f1674cb7baf67b71e5f1d69c9bb1ac4c82c Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 12 Apr 2024 16:42:18 +0900 Subject: [PATCH 066/205] fix: spend dag double spend links --- sn_client/src/audit/spend_dag.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/sn_client/src/audit/spend_dag.rs b/sn_client/src/audit/spend_dag.rs index b5bcfac819..0c0e83bab8 100644 --- a/sn_client/src/audit/spend_dag.rs +++ b/sn_client/src/audit/spend_dag.rs @@ -196,10 +196,21 @@ impl SpendDag { }); // link to ancestor - for idx in spends_at_addr.indexes() { - let ancestor_idx = NodeIndex::new(idx); - self.dag - .update_edge(ancestor_idx, new_node_idx, *spend_amount); + match spends_at_addr { + DagEntry::Spend(_, idx) | DagEntry::NotGatheredYet(idx) => { + let ancestor_idx = NodeIndex::new(*idx); + self.dag + .update_edge(ancestor_idx, new_node_idx, *spend_amount); + } + DagEntry::DoubleSpend(multiple_ancestors) => { + for (ancestor_spend, ancestor_idx) in multiple_ancestors { + if ancestor_spend.spend.spent_tx.hash() == spend.spend.parent_tx.hash() { + let ancestor_idx = NodeIndex::new(*ancestor_idx); + self.dag + .update_edge(ancestor_idx, new_node_idx, *spend_amount); + } + } + } } } From c69e5de99f422d9f898b4b8cb6bffccda850ecbb Mon Sep 17 00:00:00 2001 From: grumbach Date: Mon, 15 Apr 2024 22:36:51 +0900 Subject: [PATCH 067/205] feat: double spend fork detection, fix invalid edges issue --- sn_client/src/audit/spend_dag.rs | 107 +++++++++++++++++++++++++++++-- sn_client/src/audit/tests/mod.rs | 101 +++++++++++++++++++++++++++++ 2 files changed, 201 insertions(+), 7 deletions(-) diff --git a/sn_client/src/audit/spend_dag.rs b/sn_client/src/audit/spend_dag.rs index 0c0e83bab8..974c15e6ec 100644 --- a/sn_client/src/audit/spend_dag.rs +++ b/sn_client/src/audit/spend_dag.rs @@ -8,6 +8,7 @@ use petgraph::dot::Dot; use petgraph::graph::{DiGraph, NodeIndex}; +use petgraph::visit::EdgeRef; use serde::{Deserialize, Serialize}; use sn_transfers::{is_genesis_spend, CashNoteRedemption, NanoTokens, SignedSpend, SpendAddress}; use std::collections::{BTreeMap, BTreeSet}; @@ -130,7 +131,9 @@ impl SpendDag { Some(DagEntry::NotGatheredYet(idx)) => { self.spends .insert(spend_addr, DagEntry::Spend(Box::new(spend.clone()), idx)); - NodeIndex::new(idx) + let node_idx = NodeIndex::new(idx); + self.reset_edges(node_idx); + node_idx } // or upgrade spend to double spend if it is different from the existing one Some(DagEntry::Spend(s, idx)) => { @@ -185,7 +188,7 @@ impl SpendDag { } // link to ancestors - let spend_amount = spend.token(); + const PENDING_AMOUNT: NanoTokens = NanoTokens::from(0); for ancestor in spend.spend.parent_tx.inputs.iter() { let ancestor_addr = SpendAddress::from_unique_pubkey(&ancestor.unique_pubkey); @@ -197,17 +200,38 @@ impl SpendDag { // link to ancestor match spends_at_addr { - DagEntry::Spend(_, idx) | DagEntry::NotGatheredYet(idx) => { + DagEntry::NotGatheredYet(idx) => { let ancestor_idx = NodeIndex::new(*idx); self.dag - .update_edge(ancestor_idx, new_node_idx, *spend_amount); + .update_edge(ancestor_idx, new_node_idx, PENDING_AMOUNT); + } + DagEntry::Spend(ancestor_spend, idx) => { + let ancestor_idx = NodeIndex::new(*idx); + let ancestor_given_amount = ancestor_spend + .spend + .spent_tx + .outputs + .iter() + .find(|o| o.unique_pubkey == spend.spend.unique_pubkey) + .map(|o| o.amount) + .unwrap_or(PENDING_AMOUNT); + self.dag + .update_edge(ancestor_idx, new_node_idx, ancestor_given_amount); } DagEntry::DoubleSpend(multiple_ancestors) => { for (ancestor_spend, ancestor_idx) in multiple_ancestors { if ancestor_spend.spend.spent_tx.hash() == spend.spend.parent_tx.hash() { let ancestor_idx = NodeIndex::new(*ancestor_idx); + let ancestor_given_amount = ancestor_spend + .spend + .spent_tx + .outputs + .iter() + .find(|o| o.unique_pubkey == spend.spend.unique_pubkey) + .map(|o| o.amount) + .unwrap_or(PENDING_AMOUNT); self.dag - .update_edge(ancestor_idx, new_node_idx, *spend_amount); + .update_edge(ancestor_idx, new_node_idx, ancestor_given_amount); } } } @@ -329,6 +353,23 @@ impl SpendDag { Ok(royalties) } + /// Remove all edges from a Node in the DAG + fn reset_edges(&mut self, node: NodeIndex) { + let incoming: Vec<_> = self + .dag + .edges_directed(node, petgraph::Direction::Incoming) + .map(|e| e.id()) + .collect(); + let outgoing: Vec<_> = self + .dag + .edges_directed(node, petgraph::Direction::Outgoing) + .map(|e| e.id()) + .collect(); + for edge in incoming.into_iter().chain(outgoing.into_iter()) { + self.dag.remove_edge(edge); + } + } + /// helper that returns the direct ancestors of a given spend /// along with any faults detected /// On error returns the address of the missing ancestor @@ -467,6 +508,44 @@ impl SpendDag { Ok(recorded_faults) } + /// Checks if a double spend has multiple living descendant branches that fork + fn double_spend_has_forking_descendant_branches(&self, spends: &Vec<&SignedSpend>) -> bool { + // gather all living descendants for each branch + let mut set_of_living_descendants: BTreeSet> = BTreeSet::new(); + for spend in spends { + let gathered_descendants = spend + .spend + .spent_tx + .outputs + .iter() + .map(|o| SpendAddress::from_unique_pubkey(&o.unique_pubkey)) + .filter_map(|a| self.spends.get(&a)) + .filter_map(|s| { + if matches!(s, DagEntry::NotGatheredYet(_)) { + None + } else { + Some(s.spends()) + } + }) + .flatten() + .collect::>(); + set_of_living_descendants.insert(gathered_descendants); + } + + // make sure there is no fork + for set1 in set_of_living_descendants.iter() { + for set2 in set_of_living_descendants.iter() { + if set1.is_subset(set2) || set2.is_subset(set1) { + continue; + } else { + return true; + } + } + } + + false + } + /// Verify the DAG and record faults in the DAG /// If the DAG is invalid, return an error immediately, without mutating the DAG pub fn record_faults(&mut self, source: &SpendAddress) -> Result<(), DagError> { @@ -529,12 +608,26 @@ impl SpendDag { .map(|o| SpendAddress::from_unique_pubkey(&o.unique_pubkey)) .collect(); debug!("Making the direct descendants of the double spend at {addr:?} as faulty: {direct_descendants:?}"); - for a in direct_descendants { + for a in direct_descendants.iter() { recorded_faults.insert(SpendFault::DoubleSpentAncestor { - addr: a, + addr: *a, ancestor: *addr, }); } + if self.double_spend_has_forking_descendant_branches(&spends) { + debug!("Double spend at {addr:?} has multiple living descendant branches, poisoning them..."); + let poison = format!( + "spend is on one of multiple branches of a double spent ancestor: {addr:?}" + ); + let direct_living_descendant_spends: BTreeSet<_> = direct_descendants + .iter() + .filter_map(|a| self.spends.get(a)) + .flat_map(|s| s.spends()) + .collect(); + for s in direct_living_descendant_spends { + recorded_faults.extend(self.poison_all_descendants(s, poison.clone())?); + } + } continue; } diff --git a/sn_client/src/audit/tests/mod.rs b/sn_client/src/audit/tests/mod.rs index 2f4933d4cd..0e77ae1b25 100644 --- a/sn_client/src/audit/tests/mod.rs +++ b/sn_client/src/audit/tests/mod.rs @@ -127,6 +127,107 @@ fn test_spend_dag_double_spend_poisonning() -> Result<()> { Ok(()) } +#[test] +fn test_spend_dag_double_spend_branches() -> Result<()> { + let mut net = MockNetwork::genesis()?; + let genesis = net.genesis_spend; + + let owner1 = net.new_pk_with_balance(100)?; + let owner2 = net.new_pk_with_balance(0)?; + let owner3 = net.new_pk_with_balance(0)?; + let owner4 = net.new_pk_with_balance(0)?; + let owner5 = net.new_pk_with_balance(0)?; + let owner6 = net.new_pk_with_balance(0)?; + let owner3a = net.new_pk_with_balance(0)?; + let owner4a = net.new_pk_with_balance(0)?; + let owner5a = net.new_pk_with_balance(0)?; + + // spend normaly and save a cashnote to reuse later + net.send(&owner1, &owner2, 100)?; + let cn_to_reuse_later = net + .wallets + .get(&owner2) + .expect("owner2 wallet to exist") + .cn + .clone(); + let spend2 = net.send(&owner2, &owner3, 100)?; + let spend3 = net.send(&owner3, &owner4, 100)?; + let spend4 = net.send(&owner4, &owner5, 100)?; + let spend5 = net.send(&owner5, &owner6, 100)?; + + // reuse that cashnote to perform a double spend and create a branch + net.wallets + .get_mut(&owner2) + .expect("owner2 wallet to still exist") + .cn = cn_to_reuse_later; + let spend2a = net.send(&owner2, &owner3a, 100)?; + let spend3a = net.send(&owner3a, &owner4a, 100)?; + let spend4a = net.send(&owner4a, &owner5a, 100)?; + + // create dag + let mut dag = SpendDag::new(genesis); + for spend in net.spends { + dag.insert(spend.address(), spend.clone()); + } + assert!(dag.record_faults(&genesis).is_ok()); + // dag.dump_to_file("/tmp/test_spend_dag_double_spend_branches")?; + + // make sure double spend is detected + assert_eq!(spend2, spend2a, "both spends should be at the same address"); + let double_spent = spend2.first().expect("spend1 to have an element"); + let got = dag.get_spend_faults(double_spent); + let expected = BTreeSet::from_iter([SpendFault::DoubleSpend(*double_spent)]); + assert_eq!(got, expected, "DAG should have detected double spend"); + + // make sure the double spend's direct descendants are marked as bad + let s3 = spend3.first().expect("spend3 to have an element"); + let got = dag.get_spend_faults(s3); + let expected = BTreeSet::from_iter([SpendFault::DoubleSpentAncestor { + addr: *s3, + ancestor: *double_spent, + }]); + assert_eq!(got, expected, "spend3 should be unspendable"); + let s3a = spend3a.first().expect("spend3a to have an element"); + let got = dag.get_spend_faults(s3a); + let expected = BTreeSet::from_iter([SpendFault::DoubleSpentAncestor { + addr: *s3a, + ancestor: *double_spent, + }]); + assert_eq!(got, expected, "spend3a should be unspendable"); + + // make sur all the descendants further down the branch are marked as bad as well + let utxo_of_5a = SpendAddress::from_unique_pubkey( + &net.wallets + .get(&owner5a) + .expect("owner5a wallet to exist") + .cn + .first() + .expect("owner5a wallet to have 1 cashnote") + .unique_pubkey(), + ); + let utxo_of_6 = SpendAddress::from_unique_pubkey( + &net.wallets + .get(&owner6) + .expect("owner6 wallet to exist") + .cn + .first() + .expect("owner6 wallet to have 1 cashnote") + .unique_pubkey(), + ); + let all_descendants = [spend4, spend5, vec![utxo_of_6], spend4a, vec![utxo_of_5a]]; + for d in all_descendants.iter() { + let got = dag.get_spend_faults(d.first().expect("descendant spend to have an element")); + let expected = BTreeSet::from_iter([SpendFault::PoisonedAncestry( + *d.first().expect("d to have an element"), + format!( + "spend is on one of multiple branches of a double spent ancestor: {double_spent:?}" + ), + )]); + assert_eq!(got, expected, "all descendants should be marked as bad"); + } + Ok(()) +} + #[test] fn test_spend_dag_double_spend_detection() -> Result<()> { let mut net = MockNetwork::genesis()?; From e92b8e46bb332daa4aeddf1ec255704cd977b99e Mon Sep 17 00:00:00 2001 From: grumbach Date: Tue, 16 Apr 2024 15:38:30 +0900 Subject: [PATCH 068/205] chore: improve naming and typo fix --- sn_client/src/audit/spend_dag.rs | 4 ++-- sn_client/src/audit/tests/mod.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sn_client/src/audit/spend_dag.rs b/sn_client/src/audit/spend_dag.rs index 974c15e6ec..b92fc44ab4 100644 --- a/sn_client/src/audit/spend_dag.rs +++ b/sn_client/src/audit/spend_dag.rs @@ -132,7 +132,7 @@ impl SpendDag { self.spends .insert(spend_addr, DagEntry::Spend(Box::new(spend.clone()), idx)); let node_idx = NodeIndex::new(idx); - self.reset_edges(node_idx); + self.remove_all_edges(node_idx); node_idx } // or upgrade spend to double spend if it is different from the existing one @@ -354,7 +354,7 @@ impl SpendDag { } /// Remove all edges from a Node in the DAG - fn reset_edges(&mut self, node: NodeIndex) { + fn remove_all_edges(&mut self, node: NodeIndex) { let incoming: Vec<_> = self .dag .edges_directed(node, petgraph::Direction::Incoming) diff --git a/sn_client/src/audit/tests/mod.rs b/sn_client/src/audit/tests/mod.rs index 0e77ae1b25..620e98ed7d 100644 --- a/sn_client/src/audit/tests/mod.rs +++ b/sn_client/src/audit/tests/mod.rs @@ -195,7 +195,7 @@ fn test_spend_dag_double_spend_branches() -> Result<()> { }]); assert_eq!(got, expected, "spend3a should be unspendable"); - // make sur all the descendants further down the branch are marked as bad as well + // make sure all the descendants further down the branch are marked as bad as well let utxo_of_5a = SpendAddress::from_unique_pubkey( &net.wallets .get(&owner5a) From ca1166ea42ca50edc72ecddab3c8dd3a0fe682fd Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 16 Apr 2024 15:39:56 +0900 Subject: [PATCH 069/205] test(networking): ensure pruned data is indeed further away than kept --- sn_networking/src/record_store.rs | 153 ++++++++++++++++++------------ 1 file changed, 93 insertions(+), 60 deletions(-) diff --git a/sn_networking/src/record_store.rs b/sn_networking/src/record_store.rs index 9430081f8b..db3de731e6 100644 --- a/sn_networking/src/record_store.rs +++ b/sn_networking/src/record_store.rs @@ -359,12 +359,13 @@ impl NodeRecordStore { /// Prune the records in the store to ensure that we free up space /// for the incoming record. - fn prune_storage_if_needed_for_record(&mut self) { + /// Returns true if the record can be stored. + fn prune_records_if_needed(&mut self, incoming_record_key: &Key) -> Result<()> { let num_records = self.records.len(); // we're not full, so we don't need to prune if num_records < self.config.max_records { - return; + return Ok(()); } // sort records by distance to our local key @@ -377,6 +378,20 @@ impl NodeRecordStore { .cmp(&self.local_address.distance(&b)) }); + if let Some(last_record) = sorted_records.last() { + // if we're full and the incoming record is farther than the farthest record, we can't store it + if num_records >= self.config.max_records + && self + .local_address + .distance(&NetworkAddress::from_record_key(last_record)) + < self + .local_address + .distance(&NetworkAddress::from_record_key(incoming_record_key)) + { + return Err(Error::MaxRecords); + } + } + // sorting will be costly, hence pruning in a batch of 10 (sorted_records.len() - 10..sorted_records.len()).for_each(|i| { info!( @@ -385,6 +400,8 @@ impl NodeRecordStore { ); self.remove(&sorted_records[i]); }); + + Ok(()) } } @@ -453,7 +470,7 @@ impl NodeRecordStore { let record_key = PrettyPrintRecordKey::from(&r.key).into_owned(); trace!("PUT a verified Record: {record_key:?}"); - self.prune_storage_if_needed_for_record(); + self.prune_records_if_needed(&r.key)?; let filename = Self::generate_filename(&r.key); let file_path = self.config.storage_dir.join(&filename); @@ -1036,6 +1053,7 @@ mod tests { #[tokio::test] async fn pruning_on_full() -> Result<()> { let max_iterations = 10; + // lower max records for faster testing let max_records = 50; let temp_dir = std::env::temp_dir(); @@ -1061,9 +1079,16 @@ mod tests { network_event_sender, swarm_cmd_sender, ); - let mut stored_records: Vec = vec![]; + // keep track of everything ever stored, to check missing at the end are further away + let mut stored_records_at_some_point: Vec = vec![]; let self_address = NetworkAddress::from_peer(self_id); - for _ in 0..100 { + + // keep track of fails to assert they're further than stored + let mut failed_records = vec![]; + + // try and put an excess of records + for _ in 0..max_records * 2 { + // println!("i: {i}"); let record_key = NetworkAddress::from_peer(PeerId::random()).to_record_key(); let value = match try_serialize_record( &(0..50).map(|_| rand::random::()).collect::(), @@ -1080,68 +1105,76 @@ mod tests { }; // Will be stored anyway. - assert!(store.put_verified(record, RecordType::Chunk).is_ok()); - // We must also mark the record as stored (which would be triggered - // after the async write in nodes via NetworkEvent::CompletedWrite) - store.mark_as_stored(record_key.clone(), RecordType::Chunk); + let succeeded = store.put_verified(record, RecordType::Chunk).is_ok(); - if stored_records.len() >= max_records { - // The list is already sorted by distance, hence always shall only prune the last 10. - let mut pruned_keys = vec![]; - (0..10).for_each(|i| { - let furthest_key = stored_records.remove(stored_records.len() - 1); - println!( - "chunk {i} {:?} shall be removed", - PrettyPrintRecordKey::from(&furthest_key) - ); - pruned_keys.push(furthest_key); - }); - - for pruned_key in pruned_keys { - println!( - "record {:?} shall be pruned.", - PrettyPrintRecordKey::from(&pruned_key) - ); - // Confirm the pruned_key got removed, looping to allow async disk ops to complete. - let mut iteration = 0; - while iteration < max_iterations { - if NodeRecordStore::read_from_disk( - &store.encryption_details, - &pruned_key, - &store_config.storage_dir, - ) - .is_none() - { - break; - } - sleep(Duration::from_millis(100)).await; - iteration += 1; - } - if iteration == max_iterations { - panic!("record_store prune test failed with pruned record still exists."); + if !succeeded { + failed_records.push(record_key.clone()); + println!("failed {record_key:?}"); + } else { + // We must also mark the record as stored (which would be triggered + // after the async write in nodes via NetworkEvent::CompletedWrite) + store.mark_as_stored(record_key.clone(), RecordType::Chunk); + + println!("success sotred len: {:?} ", store.record_addresses().len()); + stored_records_at_some_point.push(record_key.clone()); + if stored_records_at_some_point.len() <= max_records { + assert!(succeeded); + } + // loop over max_iterations times to ensure async disk write had time to complete. + let mut iteration = 0; + while iteration < max_iterations { + if store.get(&record_key).is_some() { + break; } + sleep(Duration::from_millis(100)).await; + iteration += 1; } - } - - // loop over max_iterations times to ensure async disk write had time to complete. - let mut iteration = 0; - while iteration < max_iterations { - if store.get(&record_key).is_some() { - break; + if iteration == max_iterations { + panic!("record_store prune test failed with stored record {record_key:?} can't be read back"); } - sleep(Duration::from_millis(100)).await; - iteration += 1; } - if iteration == max_iterations { - panic!("record_store prune test failed with stored record cann't be read back"); + } + + let stored_data_at_end = store.record_addresses(); + assert!( + stored_data_at_end.len() == max_records, + "Stored records ({:?}) should be max_records, {max_records:?}", + stored_data_at_end.len(), + ); + + // now assert that we've stored at _least_ max records (likely many more over the liftime of the store) + assert!( + stored_records_at_some_point.len() >= max_records, + "we should have stored ata least max over time" + ); + + // now all failed records should be farther than the farthest stored record + let mut sorted_stored_data = stored_data_at_end.iter().collect_vec(); + + sorted_stored_data + .sort_by(|(a, _), (b, _)| self_address.distance(a).cmp(&self_address.distance(b))); + + // next assert that all records stored are closer than the next closest of the failed records + if let Some((most_distant_data, _)) = sorted_stored_data.last() { + for failed_record in failed_records { + let failed_data = NetworkAddress::from_record_key(&failed_record); + assert!( + self_address.distance(&failed_data) > self_address.distance(most_distant_data), + "failed record should be farther than the farthest stored record" + ); } - stored_records.push(record_key); - stored_records.sort_by(|a, b| { - let a = NetworkAddress::from_record_key(a); - let b = NetworkAddress::from_record_key(b); - self_address.distance(&a).cmp(&self_address.distance(&b)) - }); + // now for any stored data. It either shoudl still be stored OR further away than `most_distant_data` + for data in stored_records_at_some_point { + let data_addr = NetworkAddress::from_record_key(&data); + if !sorted_stored_data.contains(&(&data_addr, &RecordType::Chunk)) { + assert!( + self_address.distance(&data_addr) + > self_address.distance(most_distant_data), + "stored record should be farther than the farthest stored record" + ); + } + } } Ok(()) From 54b58ea5359b6a195b04ee2a4441fca7715ac5f6 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 16 Apr 2024 16:23:00 +0900 Subject: [PATCH 070/205] fix(record_store): prune only one record at a time --- sn_networking/src/record_store.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/sn_networking/src/record_store.rs b/sn_networking/src/record_store.rs index db3de731e6..8ab377b50e 100644 --- a/sn_networking/src/record_store.rs +++ b/sn_networking/src/record_store.rs @@ -392,14 +392,12 @@ impl NodeRecordStore { } } - // sorting will be costly, hence pruning in a batch of 10 - (sorted_records.len() - 10..sorted_records.len()).for_each(|i| { - info!( - "Record {i} {:?} will be pruned to free up space for new records", - PrettyPrintRecordKey::from(&sorted_records[i]) - ); - self.remove(&sorted_records[i]); - }); + let record_to_remove = &sorted_records[sorted_records.len() - 1]; + info!( + "Record {:?} will be pruned to free up space for new records", + PrettyPrintRecordKey::from(record_to_remove) + ); + self.remove(record_to_remove); Ok(()) } From bdca59e69aa05c653562412e1df2d493447ccf82 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 16 Apr 2024 18:50:18 +0900 Subject: [PATCH 071/205] chore(networking): notify network event on failed put due to prune --- sn_networking/src/record_store.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/sn_networking/src/record_store.rs b/sn_networking/src/record_store.rs index 8ab377b50e..10b4053b42 100644 --- a/sn_networking/src/record_store.rs +++ b/sn_networking/src/record_store.rs @@ -359,7 +359,10 @@ impl NodeRecordStore { /// Prune the records in the store to ensure that we free up space /// for the incoming record. - /// Returns true if the record can be stored. + /// Returns Ok if the record can be stored because it is closer to the local peer + /// or we are not full. + /// + /// Err MaxRecords if we cannot store as it's farther than the farthest data we have fn prune_records_if_needed(&mut self, incoming_record_key: &Key) -> Result<()> { let num_records = self.records.len(); @@ -468,7 +471,12 @@ impl NodeRecordStore { let record_key = PrettyPrintRecordKey::from(&r.key).into_owned(); trace!("PUT a verified Record: {record_key:?}"); - self.prune_records_if_needed(&r.key)?; + // notify fetcher + if let Err(error) = self.prune_records_if_needed(&r.key) { + let cmd = SwarmCmd::RemoveFailedLocalRecord { key: r.key }; + send_swarm_cmd(self.swarm_cmd_sender.clone(), cmd); + return Err(error); + } let filename = Self::generate_filename(&r.key); let file_path = self.config.storage_dir.join(&filename); From ccce41be38c14752789765678b40fabddb1caf78 Mon Sep 17 00:00:00 2001 From: Jason Paul Date: Tue, 16 Apr 2024 11:32:13 +0100 Subject: [PATCH 072/205] chore: fix typo for issue 1494 --- sn_cli/src/bin/subcommands/folders.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sn_cli/src/bin/subcommands/folders.rs b/sn_cli/src/bin/subcommands/folders.rs index d0cbf92176..30f942c2dc 100644 --- a/sn_cli/src/bin/subcommands/folders.rs +++ b/sn_cli/src/bin/subcommands/folders.rs @@ -89,7 +89,7 @@ pub(crate) async fn folders_cmds( // initialise path as a fresh new Folder with a network address derived from the root SK let root_sk = get_recovery_secret_sk(root_sk, true)?; let acc_packet = AccountPacket::init(client.clone(), root_dir, &path, &root_sk, None)?; - println!("Directoy at {path:?} initialised as a root Folder, ready to track and sync changes with the network at address: {}", acc_packet.root_folder_addr().to_hex()) + println!("Directory at {path:?} initialised as a root Folder, ready to track and sync changes with the network at address: {}", acc_packet.root_folder_addr().to_hex()) } FoldersCmds::Download { path, From 445da434b5854ee99f946de5fe6307b0eac3de56 Mon Sep 17 00:00:00 2001 From: Jason Paul Date: Mon, 15 Apr 2024 16:20:25 +0100 Subject: [PATCH 073/205] chore: remove deprecated wallet deposit cmd --- sn_cli/src/bin/main.rs | 1 - .../src/bin/subcommands/wallet/hot_wallet.rs | 26 ------------------- 2 files changed, 27 deletions(-) diff --git a/sn_cli/src/bin/main.rs b/sn_cli/src/bin/main.rs index 67294df045..62bb8fafc0 100644 --- a/sn_cli/src/bin/main.rs +++ b/sn_cli/src/bin/main.rs @@ -72,7 +72,6 @@ async fn main() -> Result<()> { if let SubCmd::Wallet(cmds) = &opt.cmd { if let WalletCmds::Address | WalletCmds::Balance { .. } - | WalletCmds::Deposit { .. } | WalletCmds::Create { .. } | WalletCmds::Sign { .. } = cmds { diff --git a/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs b/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs index a60a51d394..0641ade987 100644 --- a/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs +++ b/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs @@ -40,28 +40,6 @@ pub enum WalletCmds { #[clap(long)] peer_id: Vec, }, - /// DEPRECATED will be removed in future versions. - /// Prefer using the send and receive commands instead. - /// - /// Deposit CashNotes from the received directory to the local wallet. - /// Or Read a hex encoded CashNote from stdin. - /// - /// The default received directory is platform specific: - /// - Linux: $HOME/.local/share/safe/wallet/cash_notes - /// - macOS: $HOME/Library/Application Support/safe/wallet/cash_notes - /// - Windows: C:\Users\{username}\AppData\Roaming\safe\wallet\cash_notes - /// - /// If you find the default path unwieldy, you can also set the RECEIVED_CASHNOTES_PATH environment - /// variable to a path you would prefer to work with. - #[clap(verbatim_doc_comment)] - Deposit { - /// Read a hex encoded CashNote from stdin. - #[clap(long, default_value = "false")] - stdin: bool, - /// The hex encoded CashNote. - #[clap(long)] - cash_note: Option, - }, /// Create a hot wallet from the given (hex-encoded) key. Create { /// Hex-encoded main secret key. @@ -161,10 +139,6 @@ pub(crate) async fn wallet_cmds_without_client(cmds: &WalletCmds, root_dir: &Pat } Ok(()) } - WalletCmds::Deposit { stdin, cash_note } => { - let mut wallet = WalletApiHelper::load_from(root_dir)?; - wallet.deposit(*stdin, cash_note.as_deref()) - } WalletCmds::Create { key } => { let sk = SecretKey::from_hex(key) .map_err(|err| eyre!("Failed to parse hex-encoded SK: {err:?}"))?; From 1953acacd0880a2c5a28ff7ad72c435466019bab Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 16 Apr 2024 21:25:06 +0530 Subject: [PATCH 074/205] chore(ci): enable public rpc for wan test --- .github/workflows/nightly_wan_churn.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nightly_wan_churn.yml b/.github/workflows/nightly_wan_churn.yml index 4d51eae157..d7bb75cbf6 100644 --- a/.github/workflows/nightly_wan_churn.yml +++ b/.github/workflows/nightly_wan_churn.yml @@ -48,6 +48,7 @@ jobs: vm-count: 5 provider: digital-ocean testnet-name: NightlyChurnE2E + public-rpc: true safe-network-user: maidsafe safe-network-branch: main From 7c3cdab750c4bf7aaf01328bcff8e534de15cd94 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Fri, 29 Mar 2024 18:19:07 +0000 Subject: [PATCH 075/205] feat: make `--peer` argument optional Make the `--peer` argument on the node manager `add` command optional by handling the `PeersNotObtained` error. See the comment in the diff for more details. --- sn_node_manager/src/cmd/node.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/sn_node_manager/src/cmd/node.rs b/sn_node_manager/src/cmd/node.rs index a8352752cb..4176211083 100644 --- a/sn_node_manager/src/cmd/node.rs +++ b/sn_node_manager/src/cmd/node.rs @@ -79,10 +79,29 @@ pub async fn add( .await? }; + // Handle the `PeersNotObtained` error to make the `--peer` argument optional for the node + // manager. + // + // Since we don't use the `network-contacts` feature in the node manager, the `PeersNotObtained` + // error should occur when no `--peer` arguments were used. If the `safenode` binary we're using + // has `network-contacts` enabled (which is the case for released binaries), it's fine if the + // service definition doesn't call `safenode` with a `--peer` argument. + // + // It's simpler to not use the `network-contacts` feature in the node manager, because it can + // return a huge peer list, and that's been problematic for service definition files. + let is_first = peers.first; + let bootstrap_peers = match get_peers_from_args(peers).await { + Ok(p) => p, + Err(e) => match e { + sn_peers_acquisition::error::Error::PeersNotObtained => Vec::new(), + _ => return Err(e.into()), + }, + }; + let options = AddNodeServiceOptions { count, env_variables, - genesis: peers.first, + genesis: is_first, local, metrics_port, node_port, @@ -94,7 +113,7 @@ pub async fn add( service_log_dir_path, user: service_user, version, - bootstrap_peers: get_peers_from_args(peers).await?, + bootstrap_peers, }; add_node(options, &mut node_registry, &service_manager, verbosity).await?; From 17158d920a676f3e5837e7e39a7057133d56a280 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Fri, 29 Mar 2024 20:40:01 +0000 Subject: [PATCH 076/205] feat: provide `balance` command The node manager makes it easy to list the rewards balances for all the nodes it's managing. It was also requested that `status --details` show the balance. There was also a request to include the store cost for the node, but this information isn't as easily available. It may require an RPC extension to obtain it. --- sn_node_manager/src/bin/cli/main.rs | 18 +++++++++++ sn_node_manager/src/cmd/node.rs | 46 +++++++++++++++++++++++++---- sn_node_manager/src/lib.rs | 3 ++ 3 files changed, 62 insertions(+), 5 deletions(-) diff --git a/sn_node_manager/src/bin/cli/main.rs b/sn_node_manager/src/bin/cli/main.rs index 4a2ff8ccd0..775e4c9229 100644 --- a/sn_node_manager/src/bin/cli/main.rs +++ b/sn_node_manager/src/bin/cli/main.rs @@ -144,6 +144,20 @@ pub enum SubCmd { #[clap(long)] version: Option, }, + /// Get node reward balances. + #[clap(name = "balance")] + Balance { + /// Display the balance for a specific service using its peer ID. + /// + /// The argument can be used multiple times. + #[clap(long)] + peer_id: Vec, + /// Display the balance for a specific service using its name. + /// + /// The argument can be used multiple times. + #[clap(long, conflicts_with = "peer_id")] + service_name: Vec, + }, #[clap(subcommand)] Daemon(DaemonSubCmd), #[clap(subcommand)] @@ -594,6 +608,10 @@ async fn main() -> Result<()> { ) .await } + SubCmd::Balance { + peer_id: peer_ids, + service_name: service_names, + } => cmd::node::balance(peer_ids, service_names, verbosity).await, SubCmd::Daemon(DaemonSubCmd::Add { address, env_variables, diff --git a/sn_node_manager/src/cmd/node.rs b/sn_node_manager/src/cmd/node.rs index 4176211083..1ecf8692eb 100644 --- a/sn_node_manager/src/cmd/node.rs +++ b/sn_node_manager/src/cmd/node.rs @@ -30,6 +30,7 @@ use sn_service_management::{ rpc::RpcClient, NodeRegistry, NodeService, ServiceStatus, UpgradeOptions, UpgradeResult, }; +use sn_transfers::HotWallet; use std::{net::Ipv4Addr, path::PathBuf, str::FromStr}; pub async fn add( @@ -123,6 +124,41 @@ pub async fn add( Ok(()) } +pub async fn balance( + peer_ids: Vec, + service_names: Vec, + verbosity: VerbosityLevel, +) -> Result<()> { + if verbosity != VerbosityLevel::Minimal { + println!("================================================="); + println!(" Reward Balances "); + println!("================================================="); + } + + let mut node_registry = NodeRegistry::load(&config::get_node_registry_path()?)?; + refresh_node_registry(&mut node_registry, &ServiceController {}, true).await?; + + let service_indices = get_services_for_ops(&node_registry, peer_ids, service_names)?; + if service_indices.is_empty() { + // This could be the case if all services are at `Removed` status. + println!("No balances to display"); + return Ok(()); + } + + for &index in &service_indices { + let node = &mut node_registry.nodes[index]; + let rpc_client = RpcClient::from_socket_addr(node.rpc_socket_addr); + let service = NodeService::new(node, Box::new(rpc_client)); + let wallet = HotWallet::load_from(&service.service_data.data_dir_path)?; + println!( + "{}: {}", + service.service_data.service_name, + wallet.balance() + ); + } + Ok(()) +} + pub async fn remove( keep_directories: bool, peer_ids: Vec, @@ -143,7 +179,7 @@ pub async fn remove( let mut node_registry = NodeRegistry::load(&config::get_node_registry_path()?)?; refresh_node_registry(&mut node_registry, &ServiceController {}, true).await?; - let service_indices = get_services_to_update(&node_registry, peer_ids, service_names)?; + let service_indices = get_services_for_ops(&node_registry, peer_ids, service_names)?; if service_indices.is_empty() { // This could be the case if all services are at `Removed` status. println!("No services were eligible for removal"); @@ -187,7 +223,7 @@ pub async fn start( let mut node_registry = NodeRegistry::load(&config::get_node_registry_path()?)?; refresh_node_registry(&mut node_registry, &ServiceController {}, true).await?; - let service_indices = get_services_to_update(&node_registry, peer_ids, service_names)?; + let service_indices = get_services_for_ops(&node_registry, peer_ids, service_names)?; if service_indices.is_empty() { // This could be the case if all services are at `Removed` status. println!("No services were eligible to be started"); @@ -271,7 +307,7 @@ pub async fn stop( let mut node_registry = NodeRegistry::load(&config::get_node_registry_path()?)?; refresh_node_registry(&mut node_registry, &ServiceController {}, true).await?; - let service_indices = get_services_to_update(&node_registry, peer_ids, service_names)?; + let service_indices = get_services_for_ops(&node_registry, peer_ids, service_names)?; if service_indices.is_empty() { // This could be the case if all services are at `Removed` status. println!("No services were eligible to be stopped"); @@ -337,7 +373,7 @@ pub async fn upgrade( } } - let service_indices = get_services_to_update(&node_registry, peer_ids, service_names)?; + let service_indices = get_services_for_ops(&node_registry, peer_ids, service_names)?; let mut upgrade_summary = Vec::new(); for &index in &service_indices { @@ -392,7 +428,7 @@ pub async fn upgrade( Ok(()) } -fn get_services_to_update( +fn get_services_for_ops( node_registry: &NodeRegistry, peer_ids: Vec, service_names: Vec, diff --git a/sn_node_manager/src/lib.rs b/sn_node_manager/src/lib.rs index c16c6016f7..0d55566973 100644 --- a/sn_node_manager/src/lib.rs +++ b/sn_node_manager/src/lib.rs @@ -44,6 +44,7 @@ use sn_service_management::{ NodeRegistry, NodeServiceData, ServiceStateActions, ServiceStatus, UpgradeOptions, UpgradeResult, }; +use sn_transfers::HotWallet; use tracing::debug; pub const DAEMON_DEFAULT_PORT: u16 = 12500; @@ -326,6 +327,8 @@ pub async fn status_report( .as_ref() .map_or("-".to_string(), |p| p.len().to_string()) ); + let wallet = HotWallet::load_from(&node.data_dir_path)?; + println!("Reward balance: {}", wallet.balance()); println!(); } From c94f12aba50968b514985d88d5bbff7b8081f148 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Fri, 29 Mar 2024 21:47:22 +0000 Subject: [PATCH 077/205] refactor: provide `local` subcommand A `local` subcommand is introduced for working with local networks. We observed a user make the mistake of using the `run` command to try to connect to a remote testnet, so it seems like a `local` subcommand would be useful for clarity. There are no functional changes here, it's just some reorganisation. --- sn_node_manager/src/bin/cli/main.rs | 316 ++++++++++++++-------------- 1 file changed, 163 insertions(+), 153 deletions(-) diff --git a/sn_node_manager/src/bin/cli/main.rs b/sn_node_manager/src/bin/cli/main.rs index 775e4c9229..8e930af5c9 100644 --- a/sn_node_manager/src/bin/cli/main.rs +++ b/sn_node_manager/src/bin/cli/main.rs @@ -162,65 +162,8 @@ pub enum SubCmd { Daemon(DaemonSubCmd), #[clap(subcommand)] Faucet(FaucetSubCmd), - /// Kill the running local network. - #[clap(name = "kill")] - Kill { - /// Set this flag to keep the node's data and log directories. - #[clap(long)] - keep_directories: bool, - }, - /// Join an existing local network. - /// - /// The existing network can be managed outwith the node manager. If this is the case, use the - /// `--peer` argument to specify an initial peer to connect to. - /// - /// If no `--peer` argument is supplied, the nodes will be added to the existing local network - /// being managed by the node manager. - #[clap(name = "join")] - Join { - /// Set to build the safenode and faucet binaries. - /// - /// This option requires the command run from the root of the safe_network repository. - #[clap(long)] - build: bool, - /// The number of nodes to run. - #[clap(long, default_value_t = DEFAULT_NODE_COUNT)] - count: u16, - /// Path to a faucet binary - /// - /// The path and version arguments are mutually exclusive. - #[clap(long, conflicts_with = "faucet_version")] - faucet_path: Option, - /// The version of the faucet to use. - /// - /// The version number should be in the form X.Y.Z, with no 'v' prefix. - /// - /// The version and path arguments are mutually exclusive. - #[clap(long)] - faucet_version: Option, - /// An interval applied between launching each node. - /// - /// Units are milliseconds. - #[clap(long, default_value_t = 200)] - interval: u64, - /// Path to a safenode binary - /// - /// The path and version arguments are mutually exclusive. - #[clap(long, conflicts_with = "node_version")] - node_path: Option, - /// The version of safenode to use. - /// - /// The version number should be in the form X.Y.Z, with no 'v' prefix. - /// - /// The version and path arguments are mutually exclusive. - #[clap(long)] - node_version: Option, - #[command(flatten)] - peers: PeersArgs, - /// Set to skip the network validation process - #[clap(long)] - skip_validation: bool, - }, + #[clap(subcommand)] + Local(LocalSubCmd), /// Remove safenode service(s). /// /// Either peer ID(s) or service name(s) must be supplied. @@ -242,59 +185,6 @@ pub enum SubCmd { #[clap(long)] keep_directories: bool, }, - /// Run a local network. - /// - /// This will run safenode processes on the current machine to form a local network. A faucet - /// service will also run for dispensing tokens. - /// - /// Paths can be supplied for safenode and faucet binaries, but otherwise, the latest versions - /// will be downloaded. - #[clap(name = "run")] - Run { - /// Set to build the safenode and faucet binaries. - /// - /// This option requires the command run from the root of the safe_network repository. - #[clap(long)] - build: bool, - /// Set to remove the client data directory and kill any existing local network. - #[clap(long)] - clean: bool, - /// The number of nodes to run. - #[clap(long, default_value_t = DEFAULT_NODE_COUNT)] - count: u16, - /// Path to a faucet binary. - /// - /// The path and version arguments are mutually exclusive. - #[clap(long, conflicts_with = "faucet_version", conflicts_with = "build")] - faucet_path: Option, - /// The version of the faucet to use. - /// - /// The version number should be in the form X.Y.Z, with no 'v' prefix. - /// - /// The version and path arguments are mutually exclusive. - #[clap(long, conflicts_with = "build")] - faucet_version: Option, - /// An interval applied between launching each node. - /// - /// Units are milliseconds. - #[clap(long, default_value_t = 200)] - interval: u64, - /// Path to a safenode binary - /// - /// The path and version arguments are mutually exclusive. - #[clap(long, conflicts_with = "node_version", conflicts_with = "build")] - node_path: Option, - /// The version of safenode to use. - /// - /// The version number should be in the form X.Y.Z, with no 'v' prefix. - /// - /// The version and path arguments are mutually exclusive. - #[clap(long, conflicts_with = "build")] - node_version: Option, - /// Set to skip the network validation process - #[clap(long)] - skip_validation: bool, - }, /// Start safenode service(s). /// /// If no peer ID(s) or service name(s) are supplied, all services will be started. @@ -401,7 +291,7 @@ pub enum SubCmd { }, } -/// Manage Daemon service. +/// Manage the RPC service. #[derive(Subcommand, Debug)] pub enum DaemonSubCmd { /// Add a daemon service for issuing commands via RPC. @@ -463,7 +353,7 @@ pub enum DaemonSubCmd { Stop {}, } -/// Manage faucet services. +/// Manage the faucet service. #[allow(clippy::large_enum_variant)] #[derive(Subcommand, Debug)] pub enum FaucetSubCmd { @@ -566,6 +456,124 @@ pub enum FaucetSubCmd { }, } +/// Manage local networks. +#[allow(clippy::large_enum_variant)] +#[derive(Subcommand, Debug)] +pub enum LocalSubCmd { + /// Kill the running local network. + #[clap(name = "kill")] + Kill { + /// Set this flag to keep the node's data and log directories. + #[clap(long)] + keep_directories: bool, + }, + /// Join an existing local network. + /// + /// The existing network can be managed outwith the node manager. If this is the case, use the + /// `--peer` argument to specify an initial peer to connect to. + /// + /// If no `--peer` argument is supplied, the nodes will be added to the existing local network + /// being managed by the node manager. + #[clap(name = "join")] + Join { + /// Set to build the safenode and faucet binaries. + /// + /// This option requires the command run from the root of the safe_network repository. + #[clap(long)] + build: bool, + /// The number of nodes to run. + #[clap(long, default_value_t = DEFAULT_NODE_COUNT)] + count: u16, + /// Path to a faucet binary + /// + /// The path and version arguments are mutually exclusive. + #[clap(long, conflicts_with = "faucet_version")] + faucet_path: Option, + /// The version of the faucet to use. + /// + /// The version number should be in the form X.Y.Z, with no 'v' prefix. + /// + /// The version and path arguments are mutually exclusive. + #[clap(long)] + faucet_version: Option, + /// An interval applied between launching each node. + /// + /// Units are milliseconds. + #[clap(long, default_value_t = 200)] + interval: u64, + /// Path to a safenode binary + /// + /// The path and version arguments are mutually exclusive. + #[clap(long, conflicts_with = "node_version")] + node_path: Option, + /// The version of safenode to use. + /// + /// The version number should be in the form X.Y.Z, with no 'v' prefix. + /// + /// The version and path arguments are mutually exclusive. + #[clap(long)] + node_version: Option, + #[command(flatten)] + peers: PeersArgs, + /// Set to skip the network validation process + #[clap(long)] + skip_validation: bool, + }, + /// Run a local network. + /// + /// This will run safenode processes on the current machine to form a local network. A faucet + /// service will also run for dispensing tokens. + /// + /// Paths can be supplied for safenode and faucet binaries, but otherwise, the latest versions + /// will be downloaded. + #[clap(name = "run")] + Run { + /// Set to build the safenode and faucet binaries. + /// + /// This option requires the command run from the root of the safe_network repository. + #[clap(long)] + build: bool, + /// Set to remove the client data directory and kill any existing local network. + #[clap(long)] + clean: bool, + /// The number of nodes to run. + #[clap(long, default_value_t = DEFAULT_NODE_COUNT)] + count: u16, + /// Path to a faucet binary. + /// + /// The path and version arguments are mutually exclusive. + #[clap(long, conflicts_with = "faucet_version", conflicts_with = "build")] + faucet_path: Option, + /// The version of the faucet to use. + /// + /// The version number should be in the form X.Y.Z, with no 'v' prefix. + /// + /// The version and path arguments are mutually exclusive. + #[clap(long, conflicts_with = "build")] + faucet_version: Option, + /// An interval applied between launching each node. + /// + /// Units are milliseconds. + #[clap(long, default_value_t = 200)] + interval: u64, + /// Path to a safenode binary + /// + /// The path and version arguments are mutually exclusive. + #[clap(long, conflicts_with = "node_version", conflicts_with = "build")] + node_path: Option, + /// The version of safenode to use. + /// + /// The version number should be in the form X.Y.Z, with no 'v' prefix. + /// + /// The version and path arguments are mutually exclusive. + #[clap(long, conflicts_with = "build")] + node_version: Option, + /// Set to skip the network validation process + #[clap(long)] + skip_validation: bool, + }, +} + #[tokio::main(flavor = "current_thread")] async fn main() -> Result<()> { color_eyre::install()?; @@ -662,18 +670,8 @@ async fn main() -> Result<()> { .await } }, - SubCmd::Join { - build, - count, - faucet_path, - faucet_version, - interval, - node_path, - node_version, - peers, - skip_validation: _, - } => { - cmd::local::join( + SubCmd::Local(local_command) => match local_command { + LocalSubCmd::Join { build, count, faucet_path, @@ -682,28 +680,23 @@ async fn main() -> Result<()> { node_path, node_version, peers, - true, - ) - .await - } - SubCmd::Kill { keep_directories } => cmd::local::kill(keep_directories, verbosity), - SubCmd::Remove { - keep_directories, - peer_id: peer_ids, - service_name: service_names, - } => cmd::node::remove(keep_directories, peer_ids, service_names, verbosity).await, - SubCmd::Run { - build, - clean, - count, - faucet_path, - faucet_version, - interval, - node_path, - node_version, - skip_validation: _, - } => { - cmd::local::run( + skip_validation: _, + } => { + cmd::local::join( + build, + count, + faucet_path, + faucet_version, + interval, + node_path, + node_version, + peers, + true, + ) + .await + } + LocalSubCmd::Kill { keep_directories } => cmd::local::kill(keep_directories, verbosity), + LocalSubCmd::Run { build, clean, count, @@ -712,11 +705,28 @@ async fn main() -> Result<()> { interval, node_path, node_version, - true, - verbosity, - ) - .await - } + skip_validation: _, + } => { + cmd::local::run( + build, + clean, + count, + faucet_path, + faucet_version, + interval, + node_path, + node_version, + true, + verbosity, + ) + .await + } + }, + SubCmd::Remove { + keep_directories, + peer_id: peer_ids, + service_name: service_names, + } => cmd::node::remove(keep_directories, peer_ids, service_names, verbosity).await, SubCmd::Start { interval, peer_id: peer_ids, From 8cbe5ef109126a62a38f6e3eeccb5f037c2da38e Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Fri, 29 Mar 2024 23:14:06 +0000 Subject: [PATCH 078/205] chore: `remove` cmd operates over all services Like the `start` and `stop` commands, the `remove` command is changed so that it can operate over all services if no arguments are provided. --- sn_node_manager/src/bin/cli/main.rs | 4 +++- sn_node_manager/src/cmd/node.rs | 3 --- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/sn_node_manager/src/bin/cli/main.rs b/sn_node_manager/src/bin/cli/main.rs index 8e930af5c9..775c80dd7f 100644 --- a/sn_node_manager/src/bin/cli/main.rs +++ b/sn_node_manager/src/bin/cli/main.rs @@ -166,7 +166,9 @@ pub enum SubCmd { Local(LocalSubCmd), /// Remove safenode service(s). /// - /// Either peer ID(s) or service name(s) must be supplied. + /// If no peer ID(s) or service name(s) are supplied, all services will be removed. + /// + /// Services must be stopped before they can be removed. /// /// This command must run as the root/administrative user. #[clap(name = "remove")] diff --git a/sn_node_manager/src/cmd/node.rs b/sn_node_manager/src/cmd/node.rs index 1ecf8692eb..af28b9ac8c 100644 --- a/sn_node_manager/src/cmd/node.rs +++ b/sn_node_manager/src/cmd/node.rs @@ -168,9 +168,6 @@ pub async fn remove( if !is_running_as_root() { return Err(eyre!("The remove command must run as the root user")); } - if peer_ids.is_empty() && service_names.is_empty() { - return Err(eyre!("Either a peer ID or a service name must be supplied")); - } println!("================================================="); println!(" Remove Safenode Services "); From 93c024e69b97e6b164de85cba0e7bea35bd856d9 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Fri, 29 Mar 2024 23:22:41 +0000 Subject: [PATCH 079/205] feat: provide `reset` command This command was requested in community feedback. It will remove all services, clearing out all logs and data, and also delete the node registry file, which will reset the service counter back to zero. --- sn_node_manager/src/bin/cli/main.rs | 13 ++++++++++++ sn_node_manager/src/cmd/node.rs | 32 ++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/sn_node_manager/src/bin/cli/main.rs b/sn_node_manager/src/bin/cli/main.rs index 775c80dd7f..948b6d4f66 100644 --- a/sn_node_manager/src/bin/cli/main.rs +++ b/sn_node_manager/src/bin/cli/main.rs @@ -187,6 +187,18 @@ pub enum SubCmd { #[clap(long)] keep_directories: bool, }, + /// Reset back to a clean base state. + /// + /// Stop and remove all services and delete the node registry, which will set the service + /// counter back to zero. + /// + /// This command must run as the root/administrative user. + #[clap(name = "reset")] + Reset { + /// Set to suppress the confirmation prompt. + #[clap(long, short)] + force: bool, + }, /// Start safenode service(s). /// /// If no peer ID(s) or service name(s) are supplied, all services will be started. @@ -729,6 +741,7 @@ async fn main() -> Result<()> { peer_id: peer_ids, service_name: service_names, } => cmd::node::remove(keep_directories, peer_ids, service_names, verbosity).await, + SubCmd::Reset { force } => cmd::node::reset(force, verbosity).await, SubCmd::Start { interval, peer_id: peer_ids, diff --git a/sn_node_manager/src/cmd/node.rs b/sn_node_manager/src/cmd/node.rs index af28b9ac8c..180b70b061 100644 --- a/sn_node_manager/src/cmd/node.rs +++ b/sn_node_manager/src/cmd/node.rs @@ -31,7 +31,7 @@ use sn_service_management::{ NodeRegistry, NodeService, ServiceStatus, UpgradeOptions, UpgradeResult, }; use sn_transfers::HotWallet; -use std::{net::Ipv4Addr, path::PathBuf, str::FromStr}; +use std::{io::Write, net::Ipv4Addr, path::PathBuf, str::FromStr}; pub async fn add( count: Option, @@ -201,6 +201,36 @@ pub async fn remove( summarise_any_failed_ops(failed_services, "remove") } +pub async fn reset(force: bool, verbosity: VerbosityLevel) -> Result<()> { + if !is_running_as_root() { + return Err(eyre!("The reset command must run as the root user")); + } + + println!("================================================="); + println!(" Reset Safenode Services "); + println!("================================================="); + + if !force { + println!("WARNING: all safenode services, data, and logs will be removed."); + println!("Do you wish to proceed? [y/n]"); + std::io::stdout().flush()?; + let mut input = String::new(); + std::io::stdin().read_line(&mut input)?; + if input.trim().to_lowercase() != "y" { + println!("Reset aborted"); + return Ok(()); + } + } + + stop(vec![], vec![], verbosity.clone()).await?; + remove(false, vec![], vec![], verbosity).await?; + + let node_registry_path = config::get_node_registry_path()?; + std::fs::remove_file(node_registry_path)?; + + Ok(()) +} + pub async fn start( interval: u64, peer_ids: Vec, From 2b796e5fec5d3ec2efc2fbe163c5c2f49012afe2 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Tue, 16 Apr 2024 20:00:10 +0100 Subject: [PATCH 080/205] ci: suppress release-plz instrumentation output This output makes it difficult to find things in the logs for the release workflow runs. --- .github/workflows/release.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 78ede9631f..c593365580 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -168,8 +168,12 @@ jobs: # The `release-plz` command publishes crates which had their versions bumped, and also # creates Github releases. The binaries are then attached to the releases in the # `upload-github-release-assets` target. - cargo login ${{ secrets.CRATES_IO_TOKEN }} - release-plz release --git-token ${{ secrets.VERSION_BUMP_COMMIT_PAT }} + cargo login "${{ secrets.CRATES_IO_TOKEN }}" + # The use of 'awk' suppresses the annoying instrumentation output + # that makes the log difficult to read. + release-plz release --git-token ${{ secrets.VERSION_BUMP_COMMIT_PAT }} | \ + awk '{ if (!/^\s*in release with input/ && !/^\s{4}/) print }' + just upload-github-release-assets # Now upload the 'latest' versions to S3. This can be done later because the node manager From a24f49075f36f47423ef11bb27218e60a8bd692f Mon Sep 17 00:00:00 2001 From: qima Date: Tue, 16 Apr 2024 20:33:07 +0800 Subject: [PATCH 081/205] feat(node): restrict replication fetch range when node is full --- sn_networking/src/cmd.rs | 18 ++++++- sn_networking/src/record_store.rs | 42 +++++++++-------- sn_networking/src/record_store_api.rs | 10 ++++ sn_networking/src/replication_fetcher.rs | 60 +++++++++++++++++++++--- 4 files changed, 104 insertions(+), 26 deletions(-) diff --git a/sn_networking/src/cmd.rs b/sn_networking/src/cmd.rs index 147603f560..77266309fc 100644 --- a/sn_networking/src/cmd.rs +++ b/sn_networking/src/cmd.rs @@ -13,7 +13,10 @@ use crate::{ REPLICATION_PEERS_COUNT, }; use libp2p::{ - kad::{store::RecordStore, Quorum, Record, RecordKey}, + kad::{ + store::{Error as StoreError, RecordStore}, + Quorum, Record, RecordKey, + }, swarm::dial_opts::DialOpts, Multiaddr, PeerId, }; @@ -470,6 +473,19 @@ impl SwarmDriver { .kademlia .store_mut() .put_verified(record, record_type.clone()); + + // In case the capacity reaches full, restrict replication_fetcher to + // only fetch entries not farther than the current farthest record + if let Err(StoreError::MaxRecords) = result { + let farthest = self + .swarm + .behaviour_mut() + .kademlia + .store_mut() + .get_farthest(); + self.replication_fetcher.set_farthest_on_full(farthest); + } + // No matter storing the record succeeded or not, // the entry shall be removed from the `replication_fetcher`. // In case of local store error, re-attempt will be carried out diff --git a/sn_networking/src/record_store.rs b/sn_networking/src/record_store.rs index 10b4053b42..32b6dd6529 100644 --- a/sn_networking/src/record_store.rs +++ b/sn_networking/src/record_store.rs @@ -357,6 +357,21 @@ impl NodeRecordStore { } } + // Returns the farthest record_key to self. + pub fn get_farthest(&self) -> Option { + // sort records by distance to our local key + let mut sorted_records: Vec<_> = self.records.keys().collect(); + sorted_records.sort_by(|key_a, key_b| { + let a = NetworkAddress::from_record_key(key_a); + let b = NetworkAddress::from_record_key(key_b); + self.local_address + .distance(&a) + .cmp(&self.local_address.distance(&b)) + }); + + sorted_records.last().cloned().cloned() + } + /// Prune the records in the store to ensure that we free up space /// for the incoming record. /// Returns Ok if the record can be stored because it is closer to the local peer @@ -371,36 +386,25 @@ impl NodeRecordStore { return Ok(()); } - // sort records by distance to our local key - let mut sorted_records: Vec<_> = self.records.keys().cloned().collect(); - sorted_records.sort_by(|key_a, key_b| { - let a = NetworkAddress::from_record_key(key_a); - let b = NetworkAddress::from_record_key(key_b); - self.local_address - .distance(&a) - .cmp(&self.local_address.distance(&b)) - }); - - if let Some(last_record) = sorted_records.last() { + if let Some(last_record) = self.get_farthest() { // if we're full and the incoming record is farther than the farthest record, we can't store it if num_records >= self.config.max_records && self .local_address - .distance(&NetworkAddress::from_record_key(last_record)) + .distance(&NetworkAddress::from_record_key(&last_record)) < self .local_address .distance(&NetworkAddress::from_record_key(incoming_record_key)) { return Err(Error::MaxRecords); } - } - let record_to_remove = &sorted_records[sorted_records.len() - 1]; - info!( - "Record {:?} will be pruned to free up space for new records", - PrettyPrintRecordKey::from(record_to_remove) - ); - self.remove(record_to_remove); + info!( + "Record {:?} will be pruned to free up space for new records", + PrettyPrintRecordKey::from(&last_record) + ); + self.remove(&last_record); + } Ok(()) } diff --git a/sn_networking/src/record_store_api.rs b/sn_networking/src/record_store_api.rs index 08d7074eaa..570720f9ff 100644 --- a/sn_networking/src/record_store_api.rs +++ b/sn_networking/src/record_store_api.rs @@ -150,6 +150,16 @@ impl UnifiedRecordStore { } } + pub(crate) fn get_farthest(&self) -> Option { + match self { + Self::Client(_store) => { + warn!("Calling get_farthest at Client. This should not happen"); + None + } + Self::Node(store) => store.get_farthest(), + } + } + /// Mark the record as stored in the store. /// This adds it to records set, so it can now be retrieved /// (to be done after writes are finalised) diff --git a/sn_networking/src/replication_fetcher.rs b/sn_networking/src/replication_fetcher.rs index 1f7046668a..67304035ab 100644 --- a/sn_networking/src/replication_fetcher.rs +++ b/sn_networking/src/replication_fetcher.rs @@ -10,7 +10,7 @@ use crate::target_arch::spawn; use crate::{event::NetworkEvent, target_arch::Instant}; use libp2p::{ - kad::{RecordKey, K_VALUE}, + kad::{KBucketDistance as Distance, RecordKey, K_VALUE}, PeerId, }; use sn_protocol::{storage::RecordType, NetworkAddress, PrettyPrintRecordKey}; @@ -43,6 +43,8 @@ pub(crate) struct ReplicationFetcher { event_sender: mpsc::Sender, /// ilog2 bucket distance range that the incoming key shall be fetched distance_range: Option, + /// Restrict fetch range when node is full + farthest_distance: Option, } impl ReplicationFetcher { @@ -54,6 +56,7 @@ impl ReplicationFetcher { on_going_fetches: HashMap::new(), event_sender, distance_range: None, + farthest_distance: None, } } @@ -73,10 +76,29 @@ impl ReplicationFetcher { locally_stored_keys: &HashMap, ) -> Vec<(PeerId, RecordKey)> { self.remove_stored_keys(locally_stored_keys); + let self_address = NetworkAddress::from_peer(self.self_peer_id); + let total_incoming_keys = incoming_keys.len(); + + // In case of node full, restrict fetch range + if let Some(farthest_distance) = self.farthest_distance { + let mut out_of_range_keys = vec![]; + incoming_keys.retain(|(addr, _)| { + let is_in_range = self_address.distance(addr) <= farthest_distance; + if !is_in_range { + out_of_range_keys.push(addr.clone()); + } + is_in_range + }); + + info!("Node is full, among {total_incoming_keys} incoming replications from {holder:?}, found {} beyond current farthest", out_of_range_keys.len()); + for addr in out_of_range_keys.iter() { + trace!("Node is full, the incoming record_key {addr:?} is beyond current farthest record"); + } + } let mut keys_to_fetch = vec![]; // For new data, it will be replicated out in a special replication_list of length 1. - // And we shall `fetch` that copy immediately, if it's not being fetched. + // And we shall `fetch` that copy immediately (if in range), if it's not being fetched. if incoming_keys.len() == 1 { let (record_address, record_type) = incoming_keys[0].clone(); @@ -96,11 +118,8 @@ impl ReplicationFetcher { .retain(|_, time_out| *time_out > Instant::now()); let mut out_of_range_keys = vec![]; - let total_incoming_keys = incoming_keys.len(); // Filter out those out_of_range ones among the imcoming_keys. if let Some(ref distance_range) = self.distance_range { - let self_address = NetworkAddress::from_peer(self.self_peer_id); - incoming_keys.retain(|(addr, _record_type)| { let is_in_range = self_address.distance(addr).ilog2().unwrap_or(0) <= *distance_range; @@ -113,7 +132,6 @@ impl ReplicationFetcher { if !out_of_range_keys.is_empty() { info!("Among {total_incoming_keys} incoming replications from {holder:?}, found {} out of range", out_of_range_keys.len()); - let self_address = NetworkAddress::from_peer(self.self_peer_id); for addr in out_of_range_keys.iter() { let ilog2_distance = self_address.distance(addr).ilog2(); trace!("The incoming record_key {addr:?} is out of range with ilog2_distance being {ilog2_distance:?}, do not fetch it from {holder:?}"); @@ -133,6 +151,36 @@ impl ReplicationFetcher { keys_to_fetch } + // Node is full, any fetch (ongoing or new) shall no farther than the current farthest. + pub(crate) fn set_farthest_on_full(&mut self, farthest_in: Option) { + let self_addr = NetworkAddress::from_peer(self.self_peer_id); + + let new_farthest_distance = if let Some(farthest_in) = farthest_in { + let addr = NetworkAddress::from_record_key(&farthest_in); + self_addr.distance(&addr) + } else { + return; + }; + + if let Some(old_farthest_distance) = self.farthest_distance { + if new_farthest_distance >= old_farthest_distance { + return; + } + } + + // Remove any ongoing or pending fetches that is farther than the current farthest + self.to_be_fetched.retain(|(key, _t, _), _| { + let addr = NetworkAddress::from_record_key(key); + self_addr.distance(&addr) <= new_farthest_distance + }); + self.on_going_fetches.retain(|(key, _t), _| { + let addr = NetworkAddress::from_record_key(key); + self_addr.distance(&addr) <= new_farthest_distance + }); + + self.farthest_distance = Some(new_farthest_distance); + } + // Notify the replication fetcher about a newly added Record to the node. // The corresponding key can now be removed from the replication fetcher. // Also returns the next set of keys that has to be fetched from the peer/network. From 7df004431facafb7d40e7498f3ccccf14b66da26 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Wed, 17 Apr 2024 09:32:21 +0900 Subject: [PATCH 082/205] feat(networking): clear record on valid put If we're able to store data agin, we clear the farthest distance in the replication fetcher. (The assumed sitation is that the network has grown, and we're not responsible for less data). --- sn_networking/src/cmd.rs | 33 ++++++++++++++++-------- sn_networking/src/replication_fetcher.rs | 21 ++++++++++----- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/sn_networking/src/cmd.rs b/sn_networking/src/cmd.rs index 77266309fc..550ba9c1d1 100644 --- a/sn_networking/src/cmd.rs +++ b/sn_networking/src/cmd.rs @@ -474,16 +474,27 @@ impl SwarmDriver { .store_mut() .put_verified(record, record_type.clone()); - // In case the capacity reaches full, restrict replication_fetcher to - // only fetch entries not farther than the current farthest record - if let Err(StoreError::MaxRecords) = result { - let farthest = self - .swarm - .behaviour_mut() - .kademlia - .store_mut() - .get_farthest(); - self.replication_fetcher.set_farthest_on_full(farthest); + match result { + Ok(_) => { + // We've stored the record fine, which means our replication + // fetcher should not be limited + self.replication_fetcher.clear_farthest_on_full(); + } + Err(StoreError::MaxRecords) => { + // In case the capacity reaches full, restrict replication_fetcher to + // only fetch entries not farther than the current farthest record + let farthest = self + .swarm + .behaviour_mut() + .kademlia + .store_mut() + .get_farthest(); + self.replication_fetcher.set_farthest_on_full(farthest); + } + Err(_) => { + // Nothing special to do for these errors, + // All error cases are further logged and bubbled up below + } } // No matter storing the record succeeded or not, @@ -511,7 +522,7 @@ impl SwarmDriver { } if let Err(err) = result { - error!("Cann't store verified record {record_key:?} locally: {err:?}"); + error!("Can't store verified record {record_key:?} locally: {err:?}"); cmd_string = "PutLocalRecord error"; self.log_handling(cmd_string.to_string(), start.elapsed()); return Err(err.into()); diff --git a/sn_networking/src/replication_fetcher.rs b/sn_networking/src/replication_fetcher.rs index 67304035ab..ad1ee35d46 100644 --- a/sn_networking/src/replication_fetcher.rs +++ b/sn_networking/src/replication_fetcher.rs @@ -43,8 +43,10 @@ pub(crate) struct ReplicationFetcher { event_sender: mpsc::Sender, /// ilog2 bucket distance range that the incoming key shall be fetched distance_range: Option, - /// Restrict fetch range when node is full - farthest_distance: Option, + /// Restrict fetch range to closer than this value + /// used when the node is full, but we still have "close" data coming in + /// that is _not_ closer than our farthest max record + farthest_acceptable_distance: Option, } impl ReplicationFetcher { @@ -56,7 +58,7 @@ impl ReplicationFetcher { on_going_fetches: HashMap::new(), event_sender, distance_range: None, - farthest_distance: None, + farthest_acceptable_distance: None, } } @@ -80,7 +82,7 @@ impl ReplicationFetcher { let total_incoming_keys = incoming_keys.len(); // In case of node full, restrict fetch range - if let Some(farthest_distance) = self.farthest_distance { + if let Some(farthest_distance) = self.farthest_acceptable_distance { let mut out_of_range_keys = vec![]; incoming_keys.retain(|(addr, _)| { let is_in_range = self_address.distance(addr) <= farthest_distance; @@ -151,6 +153,13 @@ impl ReplicationFetcher { keys_to_fetch } + // Node is storing and or pruning data fine, any fetch (ongoing or new) shall no longer be restricted + // by the fetcher itself (data is checked for being "close" at a higher level before keys are fed in + // to the ReplicationFetcher) + pub(crate) fn clear_farthest_on_full(&mut self) { + self.farthest_acceptable_distance = None; + } + // Node is full, any fetch (ongoing or new) shall no farther than the current farthest. pub(crate) fn set_farthest_on_full(&mut self, farthest_in: Option) { let self_addr = NetworkAddress::from_peer(self.self_peer_id); @@ -162,7 +171,7 @@ impl ReplicationFetcher { return; }; - if let Some(old_farthest_distance) = self.farthest_distance { + if let Some(old_farthest_distance) = self.farthest_acceptable_distance { if new_farthest_distance >= old_farthest_distance { return; } @@ -178,7 +187,7 @@ impl ReplicationFetcher { self_addr.distance(&addr) <= new_farthest_distance }); - self.farthest_distance = Some(new_farthest_distance); + self.farthest_acceptable_distance = Some(new_farthest_distance); } // Notify the replication fetcher about a newly added Record to the node. From 6da215f79d48500e9d2064078f5a62ecb78908a5 Mon Sep 17 00:00:00 2001 From: Jason Paul Date: Thu, 11 Apr 2024 16:59:25 +0100 Subject: [PATCH 083/205] feat: rpc restart command --- sn_node_manager/src/lib.rs | 1 + sn_node_manager/src/rpc_client.rs | 63 +++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 sn_node_manager/src/rpc_client.rs diff --git a/sn_node_manager/src/lib.rs b/sn_node_manager/src/lib.rs index 0d55566973..25dfe8a231 100644 --- a/sn_node_manager/src/lib.rs +++ b/sn_node_manager/src/lib.rs @@ -12,6 +12,7 @@ pub mod config; pub mod helpers; pub mod local; pub mod rpc; +pub mod rpc_client; #[derive(Clone, PartialEq)] pub enum VerbosityLevel { diff --git a/sn_node_manager/src/rpc_client.rs b/sn_node_manager/src/rpc_client.rs new file mode 100644 index 0000000000..eb7c5f6da7 --- /dev/null +++ b/sn_node_manager/src/rpc_client.rs @@ -0,0 +1,63 @@ +use color_eyre::eyre::bail; +use color_eyre::{eyre::eyre, Result}; +use libp2p_identity::PeerId; +use sn_service_management::safenode_manager_proto::safe_node_manager_client::SafeNodeManagerClient; +use sn_service_management::safenode_manager_proto::NodeServiceRestartRequest; +use std::net::SocketAddr; +use std::str::FromStr; +use std::time::Duration; +use tonic::transport::Channel; +use tonic::Request; + +struct DaemonRpcClient { + addr: SocketAddr, + rpc: SafeNodeManagerClient, +} + +pub async fn restart_node( + peer_ids: Vec, + rpc_server_address: SocketAddr, + retain_peer_id: bool, +) -> Result<()> { + for peer_id in peer_ids { + let str_bytes = PeerId::from_str(&peer_id)?.to_bytes(); + + let mut daemon_client = get_rpc_client(rpc_server_address).await?; + + let _response = daemon_client + .rpc + .restart_node_service(Request::new(NodeServiceRestartRequest { + peer_id: str_bytes, + delay_millis: 0, + retain_peer_id, + })) + .await + .map_err(|err| { + eyre!( + "Failed to restart node service with {peer_id:?} at {:?} with err: {err:?}", + daemon_client.addr + ) + })?; + } + Ok(()) +} + +async fn get_rpc_client(socket_addr: SocketAddr) -> Result { + let endpoint = format!("https://{socket_addr}"); + let mut attempts = 0; + loop { + if let Ok(rpc_client) = SafeNodeManagerClient::connect(endpoint.clone()).await { + let rpc_client = DaemonRpcClient { + addr: socket_addr, + rpc: rpc_client, + }; + return Ok(rpc_client); + } + attempts += 1; + println!("Could not connect to rpc {endpoint:?}. Attempts: {attempts:?}/10"); + tokio::time::sleep(Duration::from_secs(1)).await; + if attempts >= 10 { + bail!("Failed to connect to {endpoint:?} even after 10 retries"); + } + } +} From c2ca49e2e09e8548f4626000b1992ed2c87f9212 Mon Sep 17 00:00:00 2001 From: qima Date: Wed, 17 Apr 2024 20:59:41 +0800 Subject: [PATCH 084/205] chore(node): remove duplicated record_store fullness check --- sn_networking/src/record_store.rs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/sn_networking/src/record_store.rs b/sn_networking/src/record_store.rs index 32b6dd6529..25e346a8b1 100644 --- a/sn_networking/src/record_store.rs +++ b/sn_networking/src/record_store.rs @@ -379,22 +379,19 @@ impl NodeRecordStore { /// /// Err MaxRecords if we cannot store as it's farther than the farthest data we have fn prune_records_if_needed(&mut self, incoming_record_key: &Key) -> Result<()> { - let num_records = self.records.len(); - // we're not full, so we don't need to prune - if num_records < self.config.max_records { + if self.records.len() < self.config.max_records { return Ok(()); } if let Some(last_record) = self.get_farthest() { - // if we're full and the incoming record is farther than the farthest record, we can't store it - if num_records >= self.config.max_records - && self + // if the incoming record is farther than the farthest record, we can't store it + if self + .local_address + .distance(&NetworkAddress::from_record_key(&last_record)) + < self .local_address - .distance(&NetworkAddress::from_record_key(&last_record)) - < self - .local_address - .distance(&NetworkAddress::from_record_key(incoming_record_key)) + .distance(&NetworkAddress::from_record_key(incoming_record_key)) { return Err(Error::MaxRecords); } From 840af8d6612ddc4ac4bbe506306c362912266e1c Mon Sep 17 00:00:00 2001 From: Jason Paul Date: Wed, 17 Apr 2024 12:45:23 +0100 Subject: [PATCH 085/205] docs: bash included --- README.md | 176 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 123 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 508cb7546f..c5b592599a 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,8 @@ [SafenetForum.org](https://safenetforum.org/) Own your data. Share your disk space. Get paid for doing so.
-The Data on the Safe Network is Decentralised, Autonomous, and built atop of Kademlia and Libp2p.
+The Data on the Safe Network is Decentralised, Autonomous, and built atop of Kademlia and +Libp2p.
## Table of Contents @@ -15,20 +16,29 @@ The Data on the Safe Network is Decentralised, Autonomous, and built atop of Kad ### For Users -- [CLI](https://github.com/maidsafe/safe_network/blob/main/sn_cli/README.md) The Command Line Interface, allowing users to interact with the network from their terminal. -- [Node](https://github.com/maidsafe//safe_network/blob/main/sn_node/README.md) The backbone of the safe network. Nodes can be run on commodity hardware and provide storage space and validation of transactions to the network. +- [CLI](https://github.com/maidsafe/safe_network/blob/main/sn_cli/README.md) The Command Line + Interface, allowing users to interact with the network from their terminal. +- [Node](https://github.com/maidsafe//safe_network/blob/main/sn_node/README.md) The backbone of the + safe network. Nodes can be run on commodity hardware and provide storage space and validation of + transactions to the network. ### For Developers -- [Client](https://github.com/maidsafe/safe_network/blob/main/sn_client/README.md) The client APIs allowing use of the SafeNetwork to users and developers. -- [Registers](https://github.com/maidsafe/safe_network/blob/main/sn_registers/README.md) The CRDT registers structures available on the network. -- [Node Manager](https://github.com/maidsafe/safe_network/blob/main/sn_node_manager/README.md) Use to create a local network for development and testing. -- [Faucet](https://github.com/maidsafe/safe_network/blob/main/sn_faucet/README.md) The local faucet server, used to claim genesis and request tokens from the network. -- [Node RPC](https://github.com/maidsafe/safe_network/blob/main/sn_node_rpc_client/README.md) The RPC server used by the nodes to expose API calls to the outside world. +- [Client](https://github.com/maidsafe/safe_network/blob/main/sn_client/README.md) The client APIs + allowing use of the SafeNetwork to users and developers. +- [Registers](https://github.com/maidsafe/safe_network/blob/main/sn_registers/README.md) The CRDT + registers structures available on the network. +- [Node Manager](https://github.com/maidsafe/safe_network/blob/main/sn_node_manager/README.md) Use + to create a local network for development and testing. +- [Faucet](https://github.com/maidsafe/safe_network/blob/main/sn_faucet/README.md) The local faucet + server, used to claim genesis and request tokens from the network. +- [Node RPC](https://github.com/maidsafe/safe_network/blob/main/sn_node_rpc_client/README.md) The + RPC server used by the nodes to expose API calls to the outside world. #### Releases -` ./resources/scripts/bump_version.sh` will bump the version of the crates in the `Cargo.toml` files. And generate a `chore(release):` commit, which if pushed +` ./resources/scripts/bump_version.sh` will bump the version of the crates in the `Cargo.toml` +files. And generate a `chore(release):` commit, which if pushed to main will result in CI doing a full release run. ` ./resources/scripts/bump_version.sh` can also be namespaced for other release @@ -39,9 +49,12 @@ a `beta` release on any changed crates. The Safe Network uses `quic` as the default transport protocol. -The `websockets` feature is available for the `sn_networking` crate, and above, and will allow for tcp over websockets. +The `websockets` feature is available for the `sn_networking` crate, and above, and will allow for +tcp over websockets. -If building for `wasm32` then `websockets` are enabled by default as this is the only method avilable to communicate with a network as things stand. (And that network must have `websockets` enabled.) +If building for `wasm32` then `websockets` are enabled by default as this is the only method +avilable to communicate with a network as things stand. (And that network must have `websockets` +enabled.) ##### Building for wasm32 @@ -54,23 +67,32 @@ eg `await safe.get_data("/ip4/127.0.0.1/tcp/59324/ws/p2p/12D3KooWG6kyBwLVHj5hYK2 #### Browser usage -Browser usage is highly experimental, but the wasm32 target for `sn_client` _should_ work here. YMMV until stabilised. +Browser usage is highly experimental, but the wasm32 target for `sn_client` _should_ work here. +YMMV until stabilised. ### For the Technical -- [Logging](https://github.com/maidsafe/safe_network/blob/main/sn_logging/README.md) The generalised logging crate used by the safe network (backed by the tracing crate). -- [Metrics](https://github.com/maidsafe/safe_network/blob/main/metrics/README.md) The metrics crate used by the safe network. -- [Networking](https://github.com/maidsafe/safe_network/blob/main/sn_networking/README.md) The networking layer, built atop libp2p which allows nodes and clients to communicate. -- [Protocol](https://github.com/maidsafe/safe_network/blob/main/sn_protocol/README.md) The protocol used by the safe network. -- [Transfers](https://github.com/maidsafe/safe_network/blob/main/sn_transfers/README.md) The transfers crate, used to send and receive tokens on the network. -- [Peers Acquisition](https://github.com/maidsafe/safe_network/blob/main/sn_peers_acquisition/README.md) The peers peers acqisition crate, or: how the network layer discovers bootstrap peers. -- [Build Info](https://github.com/maidsafe/safe_network/blob/main/sn_build_info/README.md) Small helper used to get the build/commit versioning info for debug purposes. +- [Logging](https://github.com/maidsafe/safe_network/blob/main/sn_logging/README.md) The + generalised logging crate used by the safe network (backed by the tracing crate). +- [Metrics](https://github.com/maidsafe/safe_network/blob/main/metrics/README.md) The metrics crate + used by the safe network. +- [Networking](https://github.com/maidsafe/safe_network/blob/main/sn_networking/README.md) The + networking layer, built atop libp2p which allows nodes and clients to communicate. +- [Protocol](https://github.com/maidsafe/safe_network/blob/main/sn_protocol/README.md) The protocol + used by the safe network. +- [Transfers](https://github.com/maidsafe/safe_network/blob/main/sn_transfers/README.md) The + transfers crate, used to send and receive tokens on the network. +- [Peers Acquisition](https://github.com/maidsafe/safe_network/blob/main/sn_peers_acquisition/README.md) + The peers peers acqisition crate, or: how the network layer discovers bootstrap peers. +- [Build Info](https://github.com/maidsafe/safe_network/blob/main/sn_build_info/README.md) Small + helper used to get the build/commit versioning info for debug purposes. ## Using a Local Network We can explore the network's features by using multiple node processes to form a local network. -The latest version of [Rust](https://www.rust-lang.org/learn/get-started) should be installed. If you already have an installation, use `rustup update` to get the latest version. +The latest version of [Rust](https://www.rust-lang.org/learn/get-started) should be installed. If +you already have an installation, use `rustup update` to get the latest version. Run all the commands from the root of this repository. @@ -81,7 +103,7 @@ Follow these steps to create a local network: 1. Create the test network:
```bash -cargo run --bin safenode-manager --features local-discovery -- run --build +cargo run --bin safenode-manager --features local-discovery -- local run --build ``` 2. Verify node status:
@@ -96,11 +118,14 @@ cargo run --bin safenode-manager --features local-discovery -- status cargo run --bin safe --features local-discovery -- wallet get-faucet 127.0.0.1:8000 ``` -The node manager's `run` command starts the node processes and a faucet process, the latter of which will dispense tokens for use with the network. The `status` command should show twenty-five running nodes. The `wallet` command retrieves some tokens, which enables file uploads. +The node manager's `run` command starts the node processes and a faucet process, the latter of +which will dispense tokens for use with the network. The `status` command should show twenty-five +running nodes. The `wallet` command retrieves some tokens, which enables file uploads. ### Files -The file storage capability can be demonstrated by uploading files to the local network, then retrieving them. +The file storage capability can be demonstrated by uploading files to the local network, then +retrieving them. Upload a file or a directory: @@ -118,32 +143,45 @@ cargo run --bin safe --features local-discovery -- files download ### Folders -The folders storage capability can be demonstrated by storing folders on the network, making changes and syncing them with the stored version on the network, as well as downloading the entire folders hierarchy onto a local directory. +The folders storage capability can be demonstrated by storing folders on the network, making +changes and syncing them with the stored version on the network, as well as downloading the entire +folders hierarchy onto a local directory. -All the following commands act on the current directory by default, but since we are building the CLI binary to run it, we will have to always provide the directory we want them to act as a path argument. -When otherwise running directly an already built CLI binary, we can simply make sure we are located at the directory we want to act on without the need of providing the path as argument. +All the following commands act on the current directory by default, but since we are building the +CLI binary to run it, we will have to always provide the directory we want them to act as a path +argument. +When otherwise running directly an already built CLI binary, we can simply make sure we are located +at the directory we want to act on without the need of providing the path as argument. -Initialise a directory to then be able to track changes made on it, and sync them up with the network: +Initialise a directory to then be able to track changes made on it, and sync them up with the +network: ```bash cargo run --bin safe --features local-discovery -- folders init ``` -Make sure you made a backup copy of the "recovery secret" generated by the above command, or the one you have provided when prompted. +Make sure you made a backup copy of the "recovery secret" generated by the above command, or the +one you have provided when prompted. -If any changes are now made to files or directories within this folder (at this point all files and folders are considered new since it has just been initalised for tracking), before trying to push those changes to the network, we can get a report of the changes that have been made locally: +If any changes are now made to files or directories within this folder (at this point all files and +folders are considered new since it has just been initalised for tracking), before trying to push +those changes to the network, we can get a report of the changes that have been made locally: ```bash cargo run --bin safe --features local-discovery -- folders status ``` -We can now push all local changes made to files and directories to the network, as well as pull any changes that could have been made to the version stored on the network since last time we synced with it: +We can now push all local changes made to files and directories to the network, as well as pull any +changes that could have been made to the version stored on the network since last time we synced +with it: ```bash cargo run --bin safe --features local-discovery -- folders sync ``` -Now that's all stored on the network, you can download the folders onto any other path by providing it as the target directory to the following command (you will be prompted to enter the "recovery secret" you obtained when initialising the directory with `init` command): +Now that's all stored on the network, you can download the folders onto any other path by providing +it as the target directory to the following command (you will be prompted to enter the "recovery +secret" you obtained when initialising the directory with `init` command): ```bash cargo run --bin safe --features local-discovery -- folders download @@ -167,7 +205,8 @@ cargo run --bin safe --features local-discovery -- wallet send 2 [address] This will output a transfer as a hex string, which should be sent to the recipient out-of-band. -For demonstration purposes, copy the transfer string and use it to receive the transfer in your own wallet: +For demonstration purposes, copy the transfer string and use it to receive the transfer in your own +wallet: ``` cargo run --bin safe --features local-discovery -- wallet receive [transfer] @@ -175,31 +214,39 @@ cargo run --bin safe --features local-discovery -- wallet receive [transfer] #### Out of band transaction signing -When you want to transfer tokens from a cold storage or hardware wallet, you can create and sign the transaction offline. This is done to prevent the private key from being exposed to any online threats. -For this type of scenarios you can create a watch-only wallet (it holds only a public key) on the online device, while using a hot-wallet (which holds the secret key) on a device that is offline. The following steps are a simple guide for performing such an operation. +When you want to transfer tokens from a cold storage or hardware wallet, you can create and sign +the transaction offline. This is done to prevent the private key from being exposed to any online +threats. +For this type of scenarios you can create a watch-only wallet (it holds only a public key) on the +online device, while using a hot-wallet (which holds the secret key) on a device that is offline. +The following steps are a simple guide for performing such an operation. Steps on the online device/computer with a watch-only wallet: 1. Create a watch-only wallet using the hex-encoded public key: `cargo run --release --bin safe -- wowallet create ` -2. Deposit a cash-note, owned by the public key used above when creating, into the watch-only wallet: +2. Deposit a cash-note, owned by the public key used above when creating, into the watch-only + wallet: `cargo run --release --bin safe -- wowallet deposit --cash-note ` 3. Build an unsigned transaction: `cargo run --release --bin safe -- wowallet transaction ` -4. Copy the built unsigned Tx generated by the above command, and send it out-of-band to the desired device where the hot-wallet can be loaded. +4. Copy the built unsigned Tx generated by the above command, and send it out-of-band to the + desired device where the hot-wallet can be loaded. Steps on the offline device/computer with the corresponding hot-wallet: -5. If you still don't have a hot-wallet created, which owns the cash-notes used to build the unsigned transaction, create it with the corresponding secret key: +5. If you still don't have a hot-wallet created, which owns the cash-notes used to build the + unsigned transaction, create it with the corresponding secret key: `cargo run --release --bin safe -- wallet create ` 6. Use the hot-wallet to sign the built transaction: `cargo run --release --bin safe -- wallet sign ` -7. Copy the signed Tx generated by the above command, and send it out-of-band back to the online device. +7. Copy the signed Tx generated by the above command, and send it out-of-band back to the online + device. Steps on the online device/computer with the watch-only wallet: @@ -209,7 +256,8 @@ Steps on the online device/computer with the watch-only wallet: 9. Deposit the change cash-note to the watch-only wallet: `cargo run --release --bin safe -- wowallet deposit ` -10. Send/share the output cash-note generated by the above command at step #8 to/with the recipient. +10. Send/share the output cash-note generated by the above command at step #8 to/with the + recipient. ### Auditing @@ -236,7 +284,9 @@ In the first terminal, using the registers example, Alice creates a register: cargo run --example registers --features=local-discovery -- --user alice --reg-nickname myregister ``` -Alice can now write a message to the register and see anything written by anyone else. For example she might enter the text "hello, who's there?" which is written to the register and then shown as the "Latest value", in her terminal: +Alice can now write a message to the register and see anything written by anyone else. For example +she might enter the text "hello, who's there?" which is written to the register and then shown as +the "Latest value", in her terminal: ``` Register address: "50f4c9d55aa1f4fc19149a86e023cd189e509519788b4ad8625a1ce62932d1938cf4242e029cada768e7af0123a98c25973804d84ad397ca65cb89d6580d04ff07e5b196ea86f882b925be6ade06fc8d" @@ -264,15 +314,22 @@ Enter a blank line to receive updates, or some text to be written. ``` -For anyone else to write to the same register they need to know its xor address, so to communicate with her friend Bob, Alice needs to find a way to send it to Bob. In her terminal, this is the value starting "50f4..." in the output above. This value it will be different each time you run the example to create a register. +For anyone else to write to the same register they need to know its xor address, so to communicate +with her friend Bob, Alice needs to find a way to send it to Bob. In her terminal, this is the +value starting "50f4..." in the output above. This value it will be different each time you run the +example to create a register. -Having received the xor address, in another terminal Bob can access the same register to see the message Alice has written, and he can write back by running this command with the address received from Alice. (Note that the command should all be on one line): +Having received the xor address, in another terminal Bob can access the same register to see the +message Alice has written, and he can write back by running this command with the address received +from Alice. (Note that the command should all be on one line): ``` cargo run --example registers --features=local-discovery -- --user bob --reg-address 50f4c9d55aa1f4fc19149a86e023cd189e509519788b4ad8625a1ce62932d1938cf4242e029cada768e7af0123a98c25973804d84ad397ca65cb89d6580d04ff07e5b196ea86f882b925be6ade06fc8d ``` -After retrieving the register and displaying the message from Alice, Bob can reply and at any time, Alice or Bob can send another message and see any new messages which have been written, or enter a blank line to poll for updates. +After retrieving the register and displaying the message from Alice, Bob can reply and at any time, +Alice or Bob can send another message and see any new messages which have been written, or enter a +blank line to poll for updates. Here's Bob writing from his terminal: @@ -290,7 +347,8 @@ Alice will see Bob's message when she either enters a blank line or writes anoth ### Inspect a Register -A second example, `register_inspect` allows you to view its structure and content. To use this with the above example you again provide the address of the register. For example: +A second example, `register_inspect` allows you to view its structure and content. To use this with +the above example you again provide the address of the register. For example: ``` cargo run --example register_inspect --features=local-discovery -- --reg-address 50f4c9d55aa1f4fc19149a86e023cd189e509519788b4ad8625a1ce62932d1938cf4242e029cada768e7af0123a98c25973804d84ad397ca65cb89d6580d04ff07e5b196ea86f882b925be6ade06fc8d @@ -328,7 +386,8 @@ where a node occurs more than once. ### RPC -The node manager launches each node process with a remote procedure call (RPC) service. The workspace has a client binary that can be used to run commands against these services. +The node manager launches each node process with a remote procedure call (RPC) service. The +workspace has a client binary that can be used to run commands against these services. Run the `status` command with the `--details` flag to get the RPC port for each node: @@ -429,31 +488,42 @@ Listening to transfers notifications... (press Ctrl+C to exit) Writing cash notes to: ./royalties-cash-notes ``` -Each received cash note is written to a file in the directory above, under another directory corresponding to the public address of the recipient. +Each received cash note is written to a file in the directory above, under another directory +corresponding to the public address of the recipient. ### Tear Down When you're finished experimenting, tear down the network: -``` -cargo run --bin safenode-manager -- kill +```bash +cargo run --bin safenode-manager -- local kill ``` ## Metrics Dashboard -Use the `open-metrics` feature flag on the node / client to start an [OpenMetrics](https://github.com/OpenObservability/OpenMetrics/) exporter. The metrics are served via a webserver started at a random port. Check the log file / stdout to find the webserver URL, `Metrics server on http://127.0.0.1:xxxx/metrics` +Use the `open-metrics` feature flag on the node / client to start +an [OpenMetrics](https://github.com/OpenObservability/OpenMetrics/) exporter. The metrics are +served via a webserver started at a random port. Check the log file / stdout to find the webserver +URL, `Metrics server on http://127.0.0.1:xxxx/metrics` -The metrics can then be collected using a collector (for e.g. Prometheus) and the data can then be imported into any visualization tool (for e.g., Grafana) to be further analyzed. Refer to this [Guide](./metrics/README.md) to easily setup a dockerized Grafana dashboard to visualize the metrics. +The metrics can then be collected using a collector (for e.g. Prometheus) and the data can then be +imported into any visualization tool (for e.g., Grafana) to be further analyzed. Refer to +this [Guide](./metrics/README.md) to easily setup a dockerized Grafana dashboard to visualize the +metrics. ## Contributing -Feel free to clone and modify this project. Pull requests are welcome.
You can also visit **[The MaidSafe Forum](https://safenetforum.org/)** for discussion or if you would like to join our online community. +Feel free to clone and modify this project. Pull requests are welcome.
You can also visit * +*[The MaidSafe Forum](https://safenetforum.org/)** for discussion or if you would like to join our +online community. ### Pull Request Process 1. Please direct all pull requests to the `alpha` branch instead of the `main` branch. -1. Ensure that your commit messages clearly describe the changes you have made and use the [Conventional Commits](https://www.conventionalcommits.org/) specification. +1. Ensure that your commit messages clearly describe the changes you have made and use + the [Conventional Commits](https://www.conventionalcommits.org/) specification. ## License -This Safe Network repository is licensed under the General Public License (GPL), version 3 ([LICENSE](http://www.gnu.org/licenses/gpl-3.0.en.html)). +This Safe Network repository is licensed under the General Public License (GPL), version +3 ([LICENSE](http://www.gnu.org/licenses/gpl-3.0.en.html)). From 77df33bc463470d23a17eaa45ed4f6615a8a0514 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 17 Apr 2024 18:49:02 +0100 Subject: [PATCH 086/205] feat: provide `--path` arg for `upgrade` cmd This feature was requested in community feedback. It allows users to build a custom binary and use that for the upgrade. We use the 'force' mechanism here to cause the upgrade process to accept this binary, regardless of its version number. The user may have built it from an experimental branch which could have a version number that's less than the current version. If the user is building their own binary from source, they are likely advanced, and aware of the risks, so it seems reasonable to apply the force. --- sn_node_manager/src/bin/cli/main.rs | 7 +++++++ sn_node_manager/src/cmd/faucet.rs | 2 +- sn_node_manager/src/cmd/mod.rs | 12 +++++++++++- sn_node_manager/src/cmd/node.rs | 19 +++++++++++++++---- 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/sn_node_manager/src/bin/cli/main.rs b/sn_node_manager/src/bin/cli/main.rs index 948b6d4f66..98ba036dc4 100644 --- a/sn_node_manager/src/bin/cli/main.rs +++ b/sn_node_manager/src/bin/cli/main.rs @@ -284,6 +284,11 @@ pub enum SubCmd { /// Required if we want to downgrade, or for testing purposes. #[clap(long)] force: bool, + /// Provide a path for the safenode binary to be used by the service. + /// + /// Useful for upgrading the service using a custom built binary. + #[clap(long)] + path: Option, /// The peer ID of the service to upgrade #[clap(long)] peer_id: Vec, @@ -759,6 +764,7 @@ async fn main() -> Result<()> { SubCmd::Upgrade { do_not_start, force, + path, peer_id: peer_ids, service_name: service_names, env_variables: provided_env_variable, @@ -767,6 +773,7 @@ async fn main() -> Result<()> { } => { cmd::node::upgrade( do_not_start, + path, force, peer_ids, provided_env_variable, diff --git a/sn_node_manager/src/cmd/faucet.rs b/sn_node_manager/src/cmd/faucet.rs index bbccc3a2b6..e726023a20 100644 --- a/sn_node_manager/src/cmd/faucet.rs +++ b/sn_node_manager/src/cmd/faucet.rs @@ -161,7 +161,7 @@ pub async fn upgrade( } let (upgrade_bin_path, target_version) = - download_and_get_upgrade_bin_path(ReleaseType::Faucet, url, version).await?; + download_and_get_upgrade_bin_path(None, ReleaseType::Faucet, url, version).await?; let faucet = node_registry.faucet.as_mut().unwrap(); if !force { diff --git a/sn_node_manager/src/cmd/mod.rs b/sn_node_manager/src/cmd/mod.rs index 07e38f8c83..1ef565c6cf 100644 --- a/sn_node_manager/src/cmd/mod.rs +++ b/sn_node_manager/src/cmd/mod.rs @@ -11,7 +11,7 @@ pub mod faucet; pub mod local; pub mod node; -use crate::helpers::download_and_extract_release; +use crate::helpers::{download_and_extract_release, get_bin_version}; use color_eyre::{eyre::eyre, Result}; use colored::Colorize; use semver::Version; @@ -33,10 +33,20 @@ pub fn is_running_as_root() -> bool { } pub async fn download_and_get_upgrade_bin_path( + custom_bin_path: Option, release_type: ReleaseType, url: Option, version: Option, ) -> Result<(PathBuf, Version)> { + if let Some(path) = custom_bin_path { + println!( + "Using the supplied custom binary at {}", + path.to_string_lossy() + ); + let bin_version = get_bin_version(&path)?; + return Ok((path, bin_version.parse()?)); + } + let release_repo = ::default_config(); if let Some(version) = version { let (upgrade_bin_path, version) = diff --git a/sn_node_manager/src/cmd/node.rs b/sn_node_manager/src/cmd/node.rs index 180b70b061..da12575627 100644 --- a/sn_node_manager/src/cmd/node.rs +++ b/sn_node_manager/src/cmd/node.rs @@ -361,6 +361,7 @@ pub async fn stop( pub async fn upgrade( do_not_start: bool, + custom_bin_path: Option, force: bool, peer_ids: Vec, provided_env_variables: Option>, @@ -373,19 +374,29 @@ pub async fn upgrade( return Err(eyre!("The upgrade command must run as the root user")); } + // In the case of a custom binary, we want to force the use of it. Regardless of its version + // number, the user has probably built it for some special case. They may have not used the + // `--force` flag; if they didn't, we can just do that for them here. + let use_force = force || custom_bin_path.is_some(); + if verbosity != VerbosityLevel::Minimal { println!("================================================="); println!(" Upgrade Safenode Services "); println!("================================================="); } - let (upgrade_bin_path, target_version) = - download_and_get_upgrade_bin_path(ReleaseType::Safenode, url, version).await?; + let (upgrade_bin_path, target_version) = download_and_get_upgrade_bin_path( + custom_bin_path.clone(), + ReleaseType::Safenode, + url, + version, + ) + .await?; let mut node_registry = NodeRegistry::load(&config::get_node_registry_path()?)?; refresh_node_registry(&mut node_registry, &ServiceController {}, true).await?; - if !force { + if !use_force { let node_versions = node_registry .nodes .iter() @@ -413,7 +424,7 @@ pub async fn upgrade( let options = UpgradeOptions { bootstrap_peers: node_registry.bootstrap_peers.clone(), env_variables: env_variables.clone(), - force, + force: use_force, start_service: !do_not_start, target_bin_path: upgrade_bin_path.clone(), target_version: target_version.clone(), From bc4848ea19562a506a66c0d02d3088c93aa70de4 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 17 Apr 2024 20:32:15 +0100 Subject: [PATCH 087/205] feat: provide `--interval` arg for `upgrade` cmd This was another community feedback request. As with the `start` command, a time-based interval is applied between each node being upgraded. We will probably need something more sophisticated, but this simple mechanism should hopefully be useful for now. --- sn_node_manager/src/bin/cli/main.rs | 7 +++++++ sn_node_manager/src/cmd/node.rs | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/sn_node_manager/src/bin/cli/main.rs b/sn_node_manager/src/bin/cli/main.rs index 98ba036dc4..cbcc3939b6 100644 --- a/sn_node_manager/src/bin/cli/main.rs +++ b/sn_node_manager/src/bin/cli/main.rs @@ -284,6 +284,11 @@ pub enum SubCmd { /// Required if we want to downgrade, or for testing purposes. #[clap(long)] force: bool, + /// An interval applied between upgrading each service. + /// + /// Units are milliseconds. + #[clap(long, default_value_t = 200)] + interval: u64, /// Provide a path for the safenode binary to be used by the service. /// /// Useful for upgrading the service using a custom built binary. @@ -764,6 +769,7 @@ async fn main() -> Result<()> { SubCmd::Upgrade { do_not_start, force, + interval, path, peer_id: peer_ids, service_name: service_names, @@ -775,6 +781,7 @@ async fn main() -> Result<()> { do_not_start, path, force, + interval, peer_ids, provided_env_variable, service_names, diff --git a/sn_node_manager/src/cmd/node.rs b/sn_node_manager/src/cmd/node.rs index da12575627..52091890f1 100644 --- a/sn_node_manager/src/cmd/node.rs +++ b/sn_node_manager/src/cmd/node.rs @@ -32,6 +32,7 @@ use sn_service_management::{ }; use sn_transfers::HotWallet; use std::{io::Write, net::Ipv4Addr, path::PathBuf, str::FromStr}; +use tracing::debug; pub async fn add( count: Option, @@ -264,6 +265,7 @@ pub async fn start( let service = NodeService::new(node, Box::new(rpc_client)); let mut service_manager = ServiceManager::new(service, Box::new(ServiceController {}), verbosity.clone()); + debug!("Sleeping for {} milliseconds", interval); std::thread::sleep(std::time::Duration::from_millis(interval)); match service_manager.start().await { Ok(()) => { @@ -363,6 +365,7 @@ pub async fn upgrade( do_not_start: bool, custom_bin_path: Option, force: bool, + interval: u64, peer_ids: Vec, provided_env_variables: Option>, service_names: Vec, @@ -449,6 +452,9 @@ pub async fn upgrade( )); } } + + debug!("Sleeping for {} milliseconds", interval); + std::thread::sleep(std::time::Duration::from_millis(interval)); } node_registry.save()?; From 76ef35eecacade00e52fb012cfb690a916b530ec Mon Sep 17 00:00:00 2001 From: qima Date: Wed, 17 Apr 2024 23:39:34 +0800 Subject: [PATCH 088/205] fix(node): avoid false alert on FailedLocalRecord --- sn_networking/src/record_store.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/sn_networking/src/record_store.rs b/sn_networking/src/record_store.rs index 25e346a8b1..b5c4433d45 100644 --- a/sn_networking/src/record_store.rs +++ b/sn_networking/src/record_store.rs @@ -472,12 +472,7 @@ impl NodeRecordStore { let record_key = PrettyPrintRecordKey::from(&r.key).into_owned(); trace!("PUT a verified Record: {record_key:?}"); - // notify fetcher - if let Err(error) = self.prune_records_if_needed(&r.key) { - let cmd = SwarmCmd::RemoveFailedLocalRecord { key: r.key }; - send_swarm_cmd(self.swarm_cmd_sender.clone(), cmd); - return Err(error); - } + self.prune_records_if_needed(&r.key)?; let filename = Self::generate_filename(&r.key); let file_path = self.config.storage_dir.join(&filename); From 0fe22e81c038a24123a42e869052894bad48a826 Mon Sep 17 00:00:00 2001 From: qima Date: Wed, 17 Apr 2024 21:10:08 +0800 Subject: [PATCH 089/205] chore(node): do not reset farthest_acceptance_distance --- sn_networking/src/cmd.rs | 9 ++++++--- sn_networking/src/replication_fetcher.rs | 7 ------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/sn_networking/src/cmd.rs b/sn_networking/src/cmd.rs index 550ba9c1d1..564f6fd4b3 100644 --- a/sn_networking/src/cmd.rs +++ b/sn_networking/src/cmd.rs @@ -476,9 +476,12 @@ impl SwarmDriver { match result { Ok(_) => { - // We've stored the record fine, which means our replication - // fetcher should not be limited - self.replication_fetcher.clear_farthest_on_full(); + // `replication_fetcher.farthest_acceptable_distance` shall only get + // shrinked, instead of expanding, even with more nodes joined to share + // the responsibility. Hence no need to reset it. + // Also, as `record_store` is `prune 1 on 1 success put`, which means + // once capacity reached max_records, there is only chance of rising slowly. + // Due to the async/parrellel handling in replication_fetcher & record_store. } Err(StoreError::MaxRecords) => { // In case the capacity reaches full, restrict replication_fetcher to diff --git a/sn_networking/src/replication_fetcher.rs b/sn_networking/src/replication_fetcher.rs index ad1ee35d46..1635bd8ea4 100644 --- a/sn_networking/src/replication_fetcher.rs +++ b/sn_networking/src/replication_fetcher.rs @@ -153,13 +153,6 @@ impl ReplicationFetcher { keys_to_fetch } - // Node is storing and or pruning data fine, any fetch (ongoing or new) shall no longer be restricted - // by the fetcher itself (data is checked for being "close" at a higher level before keys are fed in - // to the ReplicationFetcher) - pub(crate) fn clear_farthest_on_full(&mut self) { - self.farthest_acceptable_distance = None; - } - // Node is full, any fetch (ongoing or new) shall no farther than the current farthest. pub(crate) fn set_farthest_on_full(&mut self, farthest_in: Option) { let self_addr = NetworkAddress::from_peer(self.self_peer_id); From 6dedb62ef1093050e9595bcd4a62e8c55a6106a4 Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 18 Apr 2024 15:19:52 +0900 Subject: [PATCH 090/205] feat: faucet donate endpoint to feed the faucet --- sn_faucet/src/faucet_server.rs | 126 +++++++++++++++++++++++++++++++-- 1 file changed, 119 insertions(+), 7 deletions(-) diff --git a/sn_faucet/src/faucet_server.rs b/sn_faucet/src/faucet_server.rs index 2818a99a3e..a23351326f 100644 --- a/sn_faucet/src/faucet_server.rs +++ b/sn_faucet/src/faucet_server.rs @@ -11,9 +11,9 @@ use crate::token_distribution; use crate::{claim_genesis, send_tokens}; use color_eyre::eyre::Result; use fs2::FileExt; -use sn_client::Client; +use sn_client::{load_faucet_wallet_from_genesis_wallet, Client}; use sn_transfers::{ - get_faucet_data_dir, wallet_lockfile_name, HotWallet, NanoTokens, WALLET_DIR_NAME, + get_faucet_data_dir, wallet_lockfile_name, HotWallet, NanoTokens, Transfer, WALLET_DIR_NAME, }; use std::path::Path; use std::{collections::HashMap, sync::Arc}; @@ -120,6 +120,84 @@ fn is_wallet_locked() -> bool { false } +async fn respond_to_donate_request( + client: Client, + transfer_str: String, + semaphore: Arc, +) -> std::result::Result { + let permit = semaphore.try_acquire(); + info!("Got donate request with: {transfer_str}"); + + // some rate limiting + if is_wallet_locked() || permit.is_err() { + warn!("Rate limited request due"); + let mut response = Response::new("Rate limited".to_string()); + *response.status_mut() = StatusCode::TOO_MANY_REQUESTS; + + // Either opening the file or locking it failed, indicating rate limiting should occur + return Ok(response); + } + + // load wallet + let mut wallet = match load_faucet_wallet_from_genesis_wallet(&client).await { + Ok(w) => w, + Err(err) => { + eprintln!("Failed to load faucet wallet: {err}"); + error!("Failed to load faucet wallet: {err}"); + let mut response = Response::new(format!("Failed to load faucet wallet: {err}")); + *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; + return Ok(response); + } + }; + + // return key is Transfer is empty + if transfer_str.is_empty() { + let address = wallet.address().to_hex(); + return Ok(Response::new(format!("Faucet wallet address: {address}"))); + } + + // parse transfer + let transfer = match Transfer::from_hex(&transfer_str) { + Ok(t) => t, + Err(err) => { + eprintln!("Failed to parse transfer: {err}"); + error!("Failed to parse transfer {transfer_str}: {err}"); + let mut response = Response::new(format!("Failed to parse transfer: {err}")); + *response.status_mut() = StatusCode::BAD_REQUEST; + return Ok(response); + } + }; + + // receive transfer + let res = client.receive(&transfer, &wallet).await; + match res { + Ok(cashnotes) => { + let old_balance = wallet.balance(); + if let Err(e) = wallet.deposit_and_store_to_disk(&cashnotes) { + eprintln!("Failed to store deposited amount: {e}"); + error!("Failed to store deposited amount: {e}"); + let mut response = Response::new(format!("Failed to store deposited amount: {e}")); + *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; + return Ok(response); + } + let new_balance = wallet.balance(); + + info!("Successfully stored cash_note to wallet dir"); + info!("Old balance: {old_balance}, new balance: {new_balance}"); + + Ok(Response::new("Thank you!".to_string())) + } + Err(err) => { + eprintln!("Failed to verify and redeem transfer: {err}"); + error!("Failed to verify and redeem transfer: {err}"); + let mut response = + Response::new(format!("Failed to verify and redeem transfer: {err}")); + *response.status_mut() = StatusCode::BAD_REQUEST; + Ok(response) + } + } +} + async fn respond_to_gift_request( client: Client, key: String, @@ -173,6 +251,10 @@ async fn startup_server(client: Client) -> Result<()> { } let gift_client = client.clone(); + let donation_client = client.clone(); + let donation_addr_client = client.clone(); + let donation_semaphore = semaphore.clone(); + let donation_addr_semaphore = semaphore.clone(); #[cfg(feature = "distribution")] let semaphore_dist = semaphore.clone(); @@ -205,17 +287,47 @@ async fn startup_server(client: Client) -> Result<()> { respond_to_gift_request(client, key, semaphore) }); + // GET /donate + let donation_addr = warp::get().and(warp::path("donate")).and_then(move || { + debug!("Donation address request"); + let client = donation_addr_client.clone(); + let semaphore = donation_addr_semaphore.clone(); + + respond_to_donate_request(client, String::new(), semaphore) + }); + + // GET /donate/transfer + let donation_route = warp::get() + .and(warp::path!("donate" / String)) + .map(|query| { + debug!("Donation request: {query}"); + query + }) + .and_then(move |transfer| { + let client = donation_client.clone(); + let semaphore = donation_semaphore.clone(); + + respond_to_donate_request(client, transfer, semaphore) + }); + println!("Starting http server listening on port 8000..."); debug!("Starting http server listening on port 8000..."); #[cfg(feature = "distribution")] - warp::serve(distribution_route.or(gift_route)) - // warp::serve(gift_route) - .run(([0, 0, 0, 0], 8000)) - .await; + warp::serve( + distribution_route + .or(donation_route) + .or(donation_addr) + .or(gift_route), + ) + // warp::serve(gift_route) + .run(([0, 0, 0, 0], 8000)) + .await; #[cfg(not(feature = "distribution"))] - warp::serve(gift_route).run(([0, 0, 0, 0], 8000)).await; + warp::serve(donation_route.or(donation_addr).or(gift_route)) + .run(([0, 0, 0, 0], 8000)) + .await; debug!("Server closed"); Ok(()) From 8fb7ed9c7a94313756600bceabb6d38cfe19b0ff Mon Sep 17 00:00:00 2001 From: qima Date: Thu, 18 Apr 2024 18:29:53 +0800 Subject: [PATCH 091/205] chore(node): optimise record_store farthest record calculation --- sn_networking/src/record_store.rs | 71 ++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/sn_networking/src/record_store.rs b/sn_networking/src/record_store.rs index b5c4433d45..4473b6ae6e 100644 --- a/sn_networking/src/record_store.rs +++ b/sn_networking/src/record_store.rs @@ -20,7 +20,7 @@ use libp2p::{ identity::PeerId, kad::{ store::{Error, RecordStore, Result}, - KBucketKey, ProviderRecord, Record, RecordKey as Key, + KBucketDistance as Distance, KBucketKey, ProviderRecord, Record, RecordKey as Key, }, }; #[cfg(feature = "open-metrics")] @@ -79,6 +79,8 @@ pub struct NodeRecordStore { encryption_details: (Aes256GcmSiv, [u8; 4]), /// Time that this record_store got started timestamp: SystemTime, + /// Farthest record to self + farthest_record: Option<(Key, Distance)>, } /// Configuration for a `DiskBackedRecordStore`. @@ -249,7 +251,7 @@ impl NodeRecordStore { }; let records = Self::update_records_from_an_existing_store(&config, &encryption_details); - let record_store = NodeRecordStore { + let mut record_store = NodeRecordStore { local_key: KBucketKey::from(local_id), local_address: NetworkAddress::from_peer(local_id), config, @@ -262,8 +264,11 @@ impl NodeRecordStore { received_payment_count, encryption_details, timestamp, + farthest_record: None, }; + record_store.farthest_record = record_store.calculate_farthest(); + record_store.flush_historic_quoting_metrics(); record_store @@ -359,17 +364,28 @@ impl NodeRecordStore { // Returns the farthest record_key to self. pub fn get_farthest(&self) -> Option { + if let Some((ref key, _distance)) = self.farthest_record { + Some(key.clone()) + } else { + None + } + } + + // Calculates the farthest record_key to self. + fn calculate_farthest(&self) -> Option<(Key, Distance)> { // sort records by distance to our local key let mut sorted_records: Vec<_> = self.records.keys().collect(); - sorted_records.sort_by(|key_a, key_b| { - let a = NetworkAddress::from_record_key(key_a); - let b = NetworkAddress::from_record_key(key_b); - self.local_address - .distance(&a) - .cmp(&self.local_address.distance(&b)) + sorted_records.sort_by_key(|key| { + let addr = NetworkAddress::from_record_key(key); + self.local_address.distance(&addr) }); - sorted_records.last().cloned().cloned() + if let Some(key) = sorted_records.last() { + let addr = NetworkAddress::from_record_key(key); + Some(((*key).clone(), self.local_address.distance(&addr))) + } else { + None + } } /// Prune the records in the store to ensure that we free up space @@ -384,11 +400,9 @@ impl NodeRecordStore { return Ok(()); } - if let Some(last_record) = self.get_farthest() { + if let Some((farthest_record, farthest_record_distance)) = self.farthest_record.clone() { // if the incoming record is farther than the farthest record, we can't store it - if self - .local_address - .distance(&NetworkAddress::from_record_key(&last_record)) + if farthest_record_distance < self .local_address .distance(&NetworkAddress::from_record_key(incoming_record_key)) @@ -398,9 +412,9 @@ impl NodeRecordStore { info!( "Record {:?} will be pruned to free up space for new records", - PrettyPrintRecordKey::from(&last_record) + PrettyPrintRecordKey::from(&farthest_record) ); - self.remove(&last_record); + self.remove(&farthest_record); } Ok(()) @@ -432,10 +446,19 @@ impl NodeRecordStore { /// in the RecordStore records set. After this it should be safe /// to return the record as stored. pub(crate) fn mark_as_stored(&mut self, key: Key, record_type: RecordType) { - let _ = self.records.insert( - key.clone(), - (NetworkAddress::from_record_key(&key), record_type), - ); + let addr = NetworkAddress::from_record_key(&key); + let _ = self + .records + .insert(key.clone(), (addr.clone(), record_type)); + + let key_distance = self.local_address.distance(&addr); + if let Some((_farthest_record, farthest_record_distance)) = self.farthest_record.clone() { + if key_distance > farthest_record_distance { + self.farthest_record = Some((key, key_distance)); + } + } else { + self.farthest_record = Some((key, key_distance)); + } } /// Prepare record bytes for storage @@ -679,6 +702,12 @@ impl RecordStore for NodeRecordStore { let _ = metric.set(self.records.len() as i64); } + if let Some((farthest_record, _)) = self.farthest_record.clone() { + if farthest_record == *k { + self.farthest_record = self.calculate_farthest(); + } + } + let filename = Self::generate_filename(k); let file_path = self.config.storage_dir.join(&filename); @@ -1111,7 +1140,7 @@ mod tests { if !succeeded { failed_records.push(record_key.clone()); - println!("failed {record_key:?}"); + println!("failed {:?}", PrettyPrintRecordKey::from(&record_key)); } else { // We must also mark the record as stored (which would be triggered // after the async write in nodes via NetworkEvent::CompletedWrite) @@ -1162,7 +1191,7 @@ mod tests { let failed_data = NetworkAddress::from_record_key(&failed_record); assert!( self_address.distance(&failed_data) > self_address.distance(most_distant_data), - "failed record should be farther than the farthest stored record" + "failed record {failed_data:?} should be farther than the farthest stored record {most_distant_data:?}" ); } From 5c6ab50cf07f6cd8de9d45e94df59a52f0abb072 Mon Sep 17 00:00:00 2001 From: qima Date: Thu, 18 Apr 2024 20:08:09 +0800 Subject: [PATCH 092/205] fix(test): quoting metrics might have live_time field changed along time --- sn_networking/src/record_store.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sn_networking/src/record_store.rs b/sn_networking/src/record_store.rs index 4473b6ae6e..1884e05786 100644 --- a/sn_networking/src/record_store.rs +++ b/sn_networking/src/record_store.rs @@ -1026,8 +1026,8 @@ mod tests { assert!(store.get(&r.key).is_none()); // Store cost should not change if no PUT has been added assert_eq!( - store.store_cost(&r.key), - store_cost_before, + store.store_cost(&r.key).0, + store_cost_before.0, "store cost should not change over unverified put" ); From c3b91c565fa8f9a879cfb2b5a84fdf1cd109e07e Mon Sep 17 00:00:00 2001 From: qima Date: Mon, 22 Apr 2024 22:41:52 +0800 Subject: [PATCH 093/205] chore(node): lower some log levels to reduce log size --- sn_networking/src/record_store.rs | 2 +- sn_networking/src/replication_fetcher.rs | 2 +- sn_node/src/node.rs | 4 ++-- sn_node/src/put_validation.rs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sn_networking/src/record_store.rs b/sn_networking/src/record_store.rs index 1884e05786..153b790948 100644 --- a/sn_networking/src/record_store.rs +++ b/sn_networking/src/record_store.rs @@ -129,7 +129,7 @@ impl NodeRecordStore { let process_entry = |entry: &DirEntry| -> _ { let path = entry.path(); if path.is_file() { - info!("Existing record found: {path:?}"); + trace!("Existing record found: {path:?}"); // if we've got a file, lets try and read it let filename = match path.file_name().and_then(|n| n.to_str()) { Some(file_name) => file_name, diff --git a/sn_networking/src/replication_fetcher.rs b/sn_networking/src/replication_fetcher.rs index 1635bd8ea4..bc1b1d270f 100644 --- a/sn_networking/src/replication_fetcher.rs +++ b/sn_networking/src/replication_fetcher.rs @@ -222,7 +222,7 @@ impl ReplicationFetcher { pub(crate) fn next_keys_to_fetch(&mut self) -> Vec<(PeerId, RecordKey)> { self.prune_expired_keys_and_slow_nodes(); - info!("Next to fetch...."); + trace!("Next to fetch...."); if self.on_going_fetches.len() >= MAX_PARALLEL_FETCH { warn!("Replication Fetcher doesn't have free fetch capacity. Currently has {} entries in queue.", diff --git a/sn_node/src/node.rs b/sn_node/src/node.rs index 9abcc2aba8..c3b979caf6 100644 --- a/sn_node/src/node.rs +++ b/sn_node/src/node.rs @@ -369,7 +369,7 @@ impl Node { } NetworkEvent::KeysToFetchForReplication(keys) => { event_header = "KeysToFetchForReplication"; - info!("Going to fetch {:?} keys for replication", keys.len()); + trace!("Going to fetch {:?} keys for replication", keys.len()); self.record_metrics(Marker::fetching_keys_for_replication(&keys)); if let Err(err) = self.fetch_replication_keys_without_wait(keys) { @@ -756,7 +756,7 @@ fn received_valid_chunk_proof( ) -> Option<()> { if let Ok(Response::Query(QueryResponse::GetChunkExistenceProof(Ok(proof)))) = resp { if expected_proof.verify(&proof) { - debug!( + trace!( "Got a valid ChunkProof of {key:?} from {peer:?}, during peer chunk proof check." ); Some(()) diff --git a/sn_node/src/put_validation.rs b/sn_node/src/put_validation.rs index 9dcc5e459b..4199f71df8 100644 --- a/sn_node/src/put_validation.rs +++ b/sn_node/src/put_validation.rs @@ -263,7 +263,7 @@ impl Node { }; // finally store the Record directly into the local storage - debug!("Storing chunk {chunk_name:?} as Record locally"); + trace!("Storing chunk {chunk_name:?} as Record locally"); self.network.put_local_record(record); self.record_metrics(Marker::ValidChunkRecordPutFromNetwork(&pretty_key)); From 8db16e5c492ecbe310b41a383f6e01dbebece992 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 4 Apr 2024 09:48:12 +0900 Subject: [PATCH 094/205] feat(neetworking): initial tcp use by default --- Cargo.lock | 131 +++++++++++++++++++++++++++++++++--- sn_networking/Cargo.toml | 26 ++++--- sn_networking/src/driver.rs | 105 ++++++++++++++++++++--------- sn_networking/src/error.rs | 3 + sn_networking/src/event.rs | 24 +++++++ 5 files changed, 238 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d4fd66573e..1cbfe514e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -353,6 +353,19 @@ dependencies = [ "syn 2.0.58", ] +[[package]] +name = "asynchronous-codec" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4057f2c32adbb2fc158e22fb38433c8e9bbf76b75a4732c7c0cbaf695fb65568" +dependencies = [ + "bytes", + "futures-sink", + "futures-util", + "memchr", + "pin-project-lite", +] + [[package]] name = "asynchronous-codec" version = "0.7.0" @@ -2658,8 +2671,10 @@ dependencies = [ "getrandom", "instant", "libp2p-allow-block-list", + "libp2p-autonat", "libp2p-connection-limits", "libp2p-core", + "libp2p-dcutr", "libp2p-dns", "libp2p-gossipsub", "libp2p-identify", @@ -2669,6 +2684,7 @@ dependencies = [ "libp2p-metrics", "libp2p-noise", "libp2p-quic", + "libp2p-relay", "libp2p-request-response", "libp2p-swarm", "libp2p-tcp", @@ -2694,6 +2710,27 @@ dependencies = [ "void", ] +[[package]] +name = "libp2p-autonat" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95151726170e41b591735bf95c42b888fe4aa14f65216a9fbf0edcc04510586" +dependencies = [ + "async-trait", + "asynchronous-codec 0.6.2", + "futures", + "futures-timer", + "instant", + "libp2p-core", + "libp2p-identity", + "libp2p-request-response", + "libp2p-swarm", + "quick-protobuf", + "quick-protobuf-codec 0.2.0", + "rand", + "tracing", +] + [[package]] name = "libp2p-connection-limits" version = "0.3.1" @@ -2734,6 +2771,29 @@ dependencies = [ "void", ] +[[package]] +name = "libp2p-dcutr" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f7bb7fa2b9e6cad9c30a6f67e3ff5c1e4b658c62b6375e35861a85f9c97bf3" +dependencies = [ + "asynchronous-codec 0.6.2", + "either", + "futures", + "futures-bounded", + "futures-timer", + "instant", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "lru 0.11.1", + "quick-protobuf", + "quick-protobuf-codec 0.2.0", + "thiserror", + "tracing", + "void", +] + [[package]] name = "libp2p-dns" version = "0.41.1" @@ -2756,7 +2816,7 @@ version = "0.46.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d665144a616dadebdc5fff186b1233488cdcd8bfb1223218ff084b6d052c94f7" dependencies = [ - "asynchronous-codec", + "asynchronous-codec 0.7.0", "base64 0.21.7", "byteorder", "bytes", @@ -2772,7 +2832,7 @@ dependencies = [ "libp2p-swarm", "prometheus-client", "quick-protobuf", - "quick-protobuf-codec", + "quick-protobuf-codec 0.3.1", "rand", "regex", "sha2", @@ -2787,7 +2847,7 @@ version = "0.44.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20499a945d2f0221fdc6269b3848892c0f370d2ee3e19c7f65a29d8f860f6126" dependencies = [ - "asynchronous-codec", + "asynchronous-codec 0.7.0", "either", "futures", "futures-bounded", @@ -2795,9 +2855,9 @@ dependencies = [ "libp2p-core", "libp2p-identity", "libp2p-swarm", - "lru", + "lru 0.12.3", "quick-protobuf", - "quick-protobuf-codec", + "quick-protobuf-codec 0.3.1", "smallvec", "thiserror", "tracing", @@ -2829,7 +2889,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cc5767727d062c4eac74dd812c998f0e488008e82cce9c33b463d38423f9ad2" dependencies = [ "arrayvec", - "asynchronous-codec", + "asynchronous-codec 0.7.0", "bytes", "either", "fnv", @@ -2841,7 +2901,7 @@ dependencies = [ "libp2p-identity", "libp2p-swarm", "quick-protobuf", - "quick-protobuf-codec", + "quick-protobuf-codec 0.3.1", "rand", "sha2", "smallvec", @@ -2881,9 +2941,11 @@ dependencies = [ "futures", "instant", "libp2p-core", + "libp2p-dcutr", "libp2p-identify", "libp2p-identity", "libp2p-kad", + "libp2p-relay", "libp2p-swarm", "pin-project", "prometheus-client", @@ -2895,7 +2957,7 @@ version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecd0545ce077f6ea5434bcb76e8d0fe942693b4380aaad0d34a358c2bd05793" dependencies = [ - "asynchronous-codec", + "asynchronous-codec 0.7.0", "bytes", "curve25519-dalek", "futures", @@ -2939,6 +3001,31 @@ dependencies = [ "tracing", ] +[[package]] +name = "libp2p-relay" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aadb213ffc8e1a6f2b9c48dcf0fc07bf370f2ea4db7981813d45e50671c8d9d" +dependencies = [ + "asynchronous-codec 0.7.0", + "bytes", + "either", + "futures", + "futures-bounded", + "futures-timer", + "instant", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "quick-protobuf", + "quick-protobuf-codec 0.3.1", + "rand", + "static_assertions", + "thiserror", + "tracing", + "void", +] + [[package]] name = "libp2p-request-response" version = "0.26.1" @@ -3147,6 +3234,15 @@ version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +[[package]] +name = "lru" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a83fb7698b3643a0e34f9ae6f2e8f0178c0fd42f8b59d493aa271ff3a5bf21" +dependencies = [ + "hashbrown 0.14.3", +] + [[package]] name = "lru" version = "0.12.3" @@ -4318,13 +4414,26 @@ dependencies = [ "byteorder", ] +[[package]] +name = "quick-protobuf-codec" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ededb1cd78531627244d51dd0c7139fbe736c7d57af0092a76f0ffb2f56e98" +dependencies = [ + "asynchronous-codec 0.6.2", + "bytes", + "quick-protobuf", + "thiserror", + "unsigned-varint 0.7.2", +] + [[package]] name = "quick-protobuf-codec" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15a0580ab32b169745d7a39db2ba969226ca16738931be152a3209b409de2474" dependencies = [ - "asynchronous-codec", + "asynchronous-codec 0.7.0", "bytes", "quick-protobuf", "thiserror", @@ -6581,6 +6690,10 @@ name = "unsigned-varint" version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" +dependencies = [ + "asynchronous-codec 0.6.2", + "bytes", +] [[package]] name = "unsigned-varint" diff --git a/sn_networking/Cargo.toml b/sn_networking/Cargo.toml index b1a019e79f..41fe226950 100644 --- a/sn_networking/Cargo.toml +++ b/sn_networking/Cargo.toml @@ -20,30 +20,34 @@ encrypt-records = [] [dependencies] -async-trait = "0.1" -bytes = { version = "1.0.1", features = ["serde"] } -futures = "~0.3.13" -hex = "~0.4.3" -hyper = { version = "0.14", features = [ - "server", - "tcp", - "http1", -], optional = true } -itertools = "~0.12.1" -custom_debug = "~0.6.1" libp2p = { version = "0.53", features = [ "tokio", "dns", "kad", + "autonat", "macros", "request-response", "cbor", "identify", + "dcutr", + "tcp", + "relay", "noise", "tcp", "yamux", "websocket", ] } +async-trait = "0.1" +bytes = { version = "1.0.1", features = ["serde"] } +futures = "~0.3.13" +hex = "~0.4.3" +hyper = { version = "0.14", features = [ + "server", + "tcp", + "http1", +], optional = true } +itertools = "~0.12.1" +custom_debug = "~0.6.1" prometheus-client = { version = "0.22", optional = true } rand = { version = "~0.8.5", features = ["small_rng"] } rayon = "1.8.0" diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index c50c848568..6101933538 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -31,6 +31,7 @@ use futures::StreamExt; #[cfg(feature = "local-discovery")] use libp2p::mdns; use libp2p::{ + autonat, identity::Keypair, kad::{self, QueryId, Quorum, Record, K_VALUE}, multiaddr::Protocol, @@ -39,7 +40,7 @@ use libp2p::{ dial_opts::{DialOpts, PeerCondition}, ConnectionId, DialError, NetworkBehaviour, StreamProtocol, Swarm, }, - Multiaddr, PeerId, Transport, + Multiaddr, PeerId, }; #[cfg(feature = "open-metrics")] use prometheus_client::registry::Registry; @@ -170,6 +171,9 @@ pub(super) struct NodeBehaviour { #[cfg(feature = "local-discovery")] pub(super) mdns: mdns::tokio::Behaviour, pub(super) identify: libp2p::identify::Behaviour, + pub(super) autonat: libp2p::autonat::Behaviour, + pub(super) dcutr: libp2p::dcutr::Behaviour, + pub(super) relay_client: libp2p::relay::client::Behaviour, } #[derive(Debug)] @@ -296,12 +300,12 @@ impl NetworkBuilder { let listen_socket_addr = listen_addr.ok_or(NetworkError::ListenAddressNotProvided)?; // Listen on QUIC - let addr_quic = Multiaddr::from(listen_socket_addr.ip()) - .with(Protocol::Udp(listen_socket_addr.port())) - .with(Protocol::QuicV1); + let addr_tcp = + Multiaddr::from(listen_socket_addr.ip()).with(Protocol::Tcp(listen_socket_addr.port())); + let _listener_id = swarm_driver .swarm - .listen_on(addr_quic) + .listen_on(addr_tcp) .expect("Multiaddr should be supported by our configured transports"); // Listen on WebSocket @@ -445,32 +449,71 @@ impl NetworkBuilder { libp2p::identify::Behaviour::new(cfg) }; - let main_transport = transport::build_transport(&self.keypair); - - let transport = if !self.local { - debug!("Preventing non-global dials"); - // Wrap upper in a transport that prevents dialing local addresses. - libp2p::core::transport::global_only::Transport::new(main_transport).boxed() - } else { - main_transport - }; - - let behaviour = NodeBehaviour { - request_response, - kademlia, - identify, - #[cfg(feature = "local-discovery")] - mdns, - }; - - #[cfg(not(target_arch = "wasm32"))] - let swarm_config = libp2p::swarm::Config::with_tokio_executor() - .with_idle_connection_timeout(CONNECTION_KEEP_ALIVE_TIMEOUT); - #[cfg(target_arch = "wasm32")] - let swarm_config = libp2p::swarm::Config::with_wasm_executor() - .with_idle_connection_timeout(CONNECTION_KEEP_ALIVE_TIMEOUT); - - let swarm = Swarm::new(transport, behaviour, peer_id, swarm_config); + // let main_transport = transport::build_transport(&self.keypair); + + // let transport = if !self.local { + // debug!("Preventing non-global dials"); + // // Wrap upper in a transport that prevents dialing local addresses. + // libp2p::core::transport::global_only::Transport::new(main_transport).boxed() + // } else { + // main_transport + // }; + + // let dcutr = libp2p::dcutr::Behaviour::new(keypair.public().to_peer_id()); + + // let behaviour = NodeBehaviour { + // request_response, + // kademlia, + // identify, + // #[cfg(feature = "local-discovery")] + // mdns, + // dcutr, + // relay_client, + // }; + + // #[cfg(not(target_arch = "wasm32"))] + // let swarm_config = libp2p::swarm::Config::with_tokio_executor() + // .with_idle_connection_timeout(CONNECTION_KEEP_ALIVE_TIMEOUT); + // #[cfg(target_arch = "wasm32")] + // let swarm_config = libp2p::swarm::Config::with_wasm_executor() + // .with_idle_connection_timeout(CONNECTION_KEEP_ALIVE_TIMEOUT); + + // let swarm = Swarm::new(transport, behaviour, peer_id, swarm_config); + // + // + let swarm = libp2p::SwarmBuilder::with_existing_identity(self.keypair.clone()) + .with_tokio() + .with_tcp( + libp2p::tcp::Config::default() + .port_reuse(true) + .nodelay(true), + libp2p::noise::Config::new, + libp2p::yamux::Config::default, + ) + .map_err(|e| NetworkError::BahviourErr(e.to_string()))? + .with_quic() + // .with_dns()? + .with_relay_client(libp2p::noise::Config::new, libp2p::yamux::Config::default) + .map_err(|e| NetworkError::BahviourErr(e.to_string()))? + .with_behaviour(|_keypair, relay_behaviour| NodeBehaviour { + relay_client: relay_behaviour, + request_response, + #[cfg(feature = "local-discovery")] + mdns, + autonat: autonat::Behaviour::new( + peer_id, + autonat::Config { + only_global_ips: false, + ..Default::default() + }, + ), + identify, + kademlia, + dcutr: libp2p::dcutr::Behaviour::new(peer_id), + }) + .map_err(|e| NetworkError::BahviourErr(e.to_string()))? + .with_swarm_config(|c| c.with_idle_connection_timeout(CONNECTION_KEEP_ALIVE_TIMEOUT)) + .build(); let bootstrap = ContinuousBootstrap::new(); let replication_fetcher = ReplicationFetcher::new(peer_id, network_event_sender.clone()); diff --git a/sn_networking/src/error.rs b/sn_networking/src/error.rs index 841c059186..cbaa4db731 100644 --- a/sn_networking/src/error.rs +++ b/sn_networking/src/error.rs @@ -183,6 +183,9 @@ pub enum NetworkError { #[error("Outgoing response has been dropped due to a conn being closed or timeout: {0}")] OutgoingResponseDropped(Response), + + #[error("Error setting up behaviour: {0}")] + BahviourErr(String), } #[cfg(test)] diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index a1ec20a448..45b0238a3e 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -54,6 +54,10 @@ pub(super) enum NodeEvent { #[cfg(feature = "local-discovery")] Mdns(Box), Identify(Box), + Autonat(Box), + Dcutr(Box), + RelayClient(Box), + RelayServer(Box), } impl From> for NodeEvent { @@ -80,6 +84,26 @@ impl From for NodeEvent { NodeEvent::Identify(Box::new(event)) } } +impl From for NodeEvent { + fn from(event: libp2p::autonat::Event) -> Self { + NodeEvent::Autonat(Box::new(event)) + } +} +impl From for NodeEvent { + fn from(event: libp2p::dcutr::Event) -> Self { + NodeEvent::Dcutr(Box::new(event)) + } +} +impl From for NodeEvent { + fn from(event: libp2p::relay::client::Event) -> Self { + NodeEvent::RelayClient(Box::new(event)) + } +} +impl From for NodeEvent { + fn from(event: libp2p::relay::Event) -> Self { + NodeEvent::RelayServer(Box::new(event)) + } +} #[derive(CustomDebug)] /// Channel to send the `Response` through. From fbe6066d1b4b5aaf04c956eed8e03c7f92e090a8 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Fri, 5 Apr 2024 10:44:11 +0900 Subject: [PATCH 095/205] chore: cleanup, add in relay server behaviour, and todo --- sn_networking/src/driver.rs | 35 ++--------------------------------- sn_networking/src/event.rs | 10 ++++++++++ 2 files changed, 12 insertions(+), 33 deletions(-) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index 6101933538..e01cce52ab 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -174,6 +174,7 @@ pub(super) struct NodeBehaviour { pub(super) autonat: libp2p::autonat::Behaviour, pub(super) dcutr: libp2p::dcutr::Behaviour, pub(super) relay_client: libp2p::relay::client::Behaviour, + pub(super) relay_server: libp2p::relay::Behaviour, } #[derive(Debug)] @@ -449,38 +450,6 @@ impl NetworkBuilder { libp2p::identify::Behaviour::new(cfg) }; - // let main_transport = transport::build_transport(&self.keypair); - - // let transport = if !self.local { - // debug!("Preventing non-global dials"); - // // Wrap upper in a transport that prevents dialing local addresses. - // libp2p::core::transport::global_only::Transport::new(main_transport).boxed() - // } else { - // main_transport - // }; - - // let dcutr = libp2p::dcutr::Behaviour::new(keypair.public().to_peer_id()); - - // let behaviour = NodeBehaviour { - // request_response, - // kademlia, - // identify, - // #[cfg(feature = "local-discovery")] - // mdns, - // dcutr, - // relay_client, - // }; - - // #[cfg(not(target_arch = "wasm32"))] - // let swarm_config = libp2p::swarm::Config::with_tokio_executor() - // .with_idle_connection_timeout(CONNECTION_KEEP_ALIVE_TIMEOUT); - // #[cfg(target_arch = "wasm32")] - // let swarm_config = libp2p::swarm::Config::with_wasm_executor() - // .with_idle_connection_timeout(CONNECTION_KEEP_ALIVE_TIMEOUT); - - // let swarm = Swarm::new(transport, behaviour, peer_id, swarm_config); - // - // let swarm = libp2p::SwarmBuilder::with_existing_identity(self.keypair.clone()) .with_tokio() .with_tcp( @@ -492,11 +461,11 @@ impl NetworkBuilder { ) .map_err(|e| NetworkError::BahviourErr(e.to_string()))? .with_quic() - // .with_dns()? .with_relay_client(libp2p::noise::Config::new, libp2p::yamux::Config::default) .map_err(|e| NetworkError::BahviourErr(e.to_string()))? .with_behaviour(|_keypair, relay_behaviour| NodeBehaviour { relay_client: relay_behaviour, + relay_server: libp2p::relay::Behaviour::new(peer_id, Default::default()), request_response, #[cfg(feature = "local-discovery")] mdns, diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 45b0238a3e..d96fdc7fda 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -260,6 +260,16 @@ impl SwarmDriver { SwarmEvent::Behaviour(NodeEvent::Identify(iden)) => { event_string = "identify"; + // TODO: in identify, we first need to know if we are behind nat. + // 1: TCP connections only, we do run autonat request in the background. + // + // (Which means all nodes must listen and act on autonat for tcp conns only.) + // + // 2. If we are behind nat and unreachable, we should not dial other nodes for now. + // 3. If we are unreachable, we connect to a relay server. + // 4. We then use dcutr to connect to other nodes and register valid quic connections. + // 5. That is holes punched? + // Match on the Identify event. match *iden { // If the event is a Received event, handle the received peer information. From 1c3866b8ea0105e0928420bffff40ac8a394fec7 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Mon, 8 Apr 2024 11:28:49 +0900 Subject: [PATCH 096/205] feat(networking): add in autonat server basics --- sn_networking/src/driver.rs | 14 +++++- sn_networking/src/event.rs | 92 +++++++++++++++++++++++++++++++------ 2 files changed, 90 insertions(+), 16 deletions(-) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index e01cce52ab..d9b4f80342 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -300,7 +300,7 @@ impl NetworkBuilder { // Listen on the provided address let listen_socket_addr = listen_addr.ok_or(NetworkError::ListenAddressNotProvided)?; - // Listen on QUIC + // Listen on TCP for autonat let addr_tcp = Multiaddr::from(listen_socket_addr.ip()).with(Protocol::Tcp(listen_socket_addr.port())); @@ -309,6 +309,15 @@ impl NetworkBuilder { .listen_on(addr_tcp) .expect("Multiaddr should be supported by our configured transports"); + // Listen on QUIC + let addr_quic = Multiaddr::from(listen_socket_addr.ip()) + .with(Protocol::Udp(listen_socket_addr.port())) + .with(Protocol::QuicV1); + let _listener_id = swarm_driver + .swarm + .listen_on(addr_quic) + .expect("Multiaddr should be supported by our configured transports"); + // Listen on WebSocket #[cfg(any(feature = "websockets", target_arch = "wasm32"))] { @@ -493,6 +502,7 @@ impl NetworkBuilder { local: self.local, listen_port: self.listen_addr.map(|addr| addr.port()), is_client, + is_known_behind_nat: false, connected_peers: 0, bootstrap, close_group: Default::default(), @@ -547,6 +557,8 @@ pub struct SwarmDriver { #[allow(unused)] pub(crate) network_metrics: NetworkMetrics, + pub(crate) is_known_behind_nat: bool, + cmd_receiver: mpsc::Receiver, event_sender: mpsc::Sender, // Use `self.send_event()` to send a NetworkEvent. diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index d96fdc7fda..ec649c4051 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -20,6 +20,7 @@ use itertools::Itertools; #[cfg(feature = "local-discovery")] use libp2p::mdns; use libp2p::{ + autonat::NatStatus, kad::{self, GetClosestPeersError, InboundRequest, QueryResult, Record, RecordKey, K_VALUE}, multiaddr::Protocol, request_response::{self, Message, ResponseChannel as PeerResponseChannel}, @@ -256,7 +257,63 @@ impl SwarmDriver { event_string = "kad_event"; self.handle_kad_event(kad_event)?; } - // Handle the Identify event from the libp2p swarm. + // Handle the autonat events from the libp2p swarm. + SwarmEvent::Behaviour(NodeEvent::Autonat(autonat_event)) => { + event_string = "autonat_event"; + + match *autonat_event { + libp2p::autonat::Event::StatusChanged { old, new } => { + info!("NAT status change... was: {old:?} now: {new:?}"); + + if !new.is_public() { + self.is_known_behind_nat = true; + warn!("BEHIND NAT>>>>>>>"); + // we are behind NAT + // do relay hole punch etc + } else if let NatStatus::Public(addr) = new { + self.is_known_behind_nat = false; + info!("NOT BEHIND NAT go server mode!!"); + + // Trigger server mode if we're not a client + if !self.is_client { + if self.local { + // all addresses are effectively external here... + // this is needed for Kad Mode::Server + self.swarm.add_external_address(addr); + } else { + // only add our global addresses + if multiaddr_is_global(&addr) { + self.swarm.add_external_address(addr); + } + } + } + } + } + libp2p::autonat::Event::InboundProbe(in_event) => { + debug!("NAT inbound probe"); + } + libp2p::autonat::Event::OutboundProbe(out_event) => match out_event { + libp2p::autonat::OutboundProbeEvent::Error { + probe_id, + peer, + error, + } => { + error!(?peer, ?error, "NAT outbound probe failed"); + } + libp2p::autonat::OutboundProbeEvent::Response { + probe_id, + peer, + address, + } => { + info!(%peer, ?address, "NAT outbound probe success"); + } + libp2p::autonat::OutboundProbeEvent::Request { probe_id, peer } => { + info!(%peer, "NAT outbound probe request"); + } + }, + } + } + SwarmEvent::Behaviour(NodeEvent::Identify(iden)) => { event_string = "identify"; @@ -341,6 +398,25 @@ impl SwarmDriver { (true, None) }; + // Add some autonat logic here + for p in info.protocols { + // TODO tie to an actual protocol version + if p.to_string().contains("autonat") { + info!(%peer_id, ?addrs, "peer supports autonat: {:?}",peer_id); + + for a in &addrs { + // add each address to the autonat server list + self.swarm + .behaviour_mut() + .autonat + .add_server(peer_id, Some(a.clone())); + } + + // break out + break; + } + } + if !kbucket_full { info!(%peer_id, ?addrs, "received identify info from undialed peer for not full kbucket {ilog2:?}, dial back to confirm external accessible"); self.dialed_peers.push(peer_id); @@ -432,20 +508,6 @@ impl SwarmDriver { let local_peer_id = *self.swarm.local_peer_id(); let address = address.with(Protocol::P2p(local_peer_id)); - // Trigger server mode if we're not a client - if !self.is_client { - if self.local { - // all addresses are effectively external here... - // this is needed for Kad Mode::Server - self.swarm.add_external_address(address.clone()); - } else { - // only add our global addresses - if multiaddr_is_global(&address) { - self.swarm.add_external_address(address.clone()); - } - } - } - self.send_event(NetworkEvent::NewListenAddr(address.clone())); info!("Local node is listening on {address:?}"); From 79278551bc28cb14c027ed6dc6edddb1d79c9bfd Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Mon, 8 Apr 2024 16:33:41 +0530 Subject: [PATCH 097/205] feat(relay): impl RelayManager to perform circuit relay when behind NAT --- sn_networking/src/cmd.rs | 3 +- sn_networking/src/driver.rs | 14 ++- sn_networking/src/event.rs | 36 +++++- sn_networking/src/lib.rs | 1 + sn_networking/src/relay_manager.rs | 172 +++++++++++++++++++++++++++++ sn_node/src/node.rs | 1 + 6 files changed, 221 insertions(+), 6 deletions(-) create mode 100644 sn_networking/src/relay_manager.rs diff --git a/sn_networking/src/cmd.rs b/sn_networking/src/cmd.rs index 564f6fd4b3..8729338a8e 100644 --- a/sn_networking/src/cmd.rs +++ b/sn_networking/src/cmd.rs @@ -588,8 +588,7 @@ impl SwarmDriver { SwarmCmd::Dial { addr, sender } => { cmd_string = "Dial"; - let mut addr_copy = addr.clone(); - if let Some(peer_id) = multiaddr_pop_p2p(&mut addr_copy) { + if let Some(peer_id) = multiaddr_pop_p2p(&mut addr.clone()) { // Only consider the dial peer is bootstrap node when proper PeerId is provided. if let Some(kbucket) = self.swarm.behaviour_mut().kademlia.kbucket(peer_id) { let ilog2 = kbucket.range().0.ilog2(); diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index d9b4f80342..89f189ea0f 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -16,16 +16,16 @@ use crate::{ circular_vec::CircularVec, cmd::SwarmCmd, error::{NetworkError, Result}, - event::NetworkEvent, - event::NodeEvent, + event::{NetworkEvent, NodeEvent}, get_record_handler::PendingGetRecord, multiaddr_pop_p2p, network_discovery::NetworkDiscovery, record_store::{ClientRecordStore, NodeRecordStore, NodeRecordStoreConfig}, record_store_api::UnifiedRecordStore, + relay_manager::RelayManager, replication_fetcher::ReplicationFetcher, target_arch::{interval, spawn, Instant}, - transport, Network, CLOSE_GROUP_SIZE, + Network, CLOSE_GROUP_SIZE, }; use futures::StreamExt; #[cfg(feature = "local-discovery")] @@ -185,6 +185,7 @@ pub struct NetworkBuilder { listen_addr: Option, request_timeout: Option, concurrency_limit: Option, + initial_peers: Vec, #[cfg(feature = "open-metrics")] metrics_registry: Option, #[cfg(feature = "open-metrics")] @@ -200,6 +201,7 @@ impl NetworkBuilder { listen_addr: None, request_timeout: None, concurrency_limit: None, + initial_peers: Default::default(), #[cfg(feature = "open-metrics")] metrics_registry: None, #[cfg(feature = "open-metrics")] @@ -219,6 +221,10 @@ impl NetworkBuilder { self.concurrency_limit = Some(concurrency_limit); } + pub fn initial_peers(&mut self, initial_peers: Vec) { + self.initial_peers = initial_peers; + } + #[cfg(feature = "open-metrics")] pub fn metrics_registry(&mut self, metrics_registry: Registry) { self.metrics_registry = Some(metrics_registry); @@ -505,6 +511,7 @@ impl NetworkBuilder { is_known_behind_nat: false, connected_peers: 0, bootstrap, + relay_manager: RelayManager::new(self.initial_peers), close_group: Default::default(), replication_fetcher, #[cfg(feature = "open-metrics")] @@ -550,6 +557,7 @@ pub struct SwarmDriver { pub(crate) listen_port: Option, pub(crate) connected_peers: usize, pub(crate) bootstrap: ContinuousBootstrap, + pub(crate) relay_manager: RelayManager, /// The peers that are closer to our PeerId. Includes self. pub(crate) close_group: Vec, pub(crate) replication_fetcher: ReplicationFetcher, diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index ec649c4051..65977786f0 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -265,11 +265,13 @@ impl SwarmDriver { libp2p::autonat::Event::StatusChanged { old, new } => { info!("NAT status change... was: {old:?} now: {new:?}"); - if !new.is_public() { + // if we are private, establish relay + if new == NatStatus::Private { self.is_known_behind_nat = true; warn!("BEHIND NAT>>>>>>>"); // we are behind NAT // do relay hole punch etc + self.relay_manager.dial_relays(&mut self.swarm); } else if let NatStatus::Public(addr) = new { self.is_known_behind_nat = false; info!("NOT BEHIND NAT go server mode!!"); @@ -313,7 +315,17 @@ impl SwarmDriver { }, } } + SwarmEvent::Behaviour(NodeEvent::RelayClient(event)) => { + event_string = "relay_client_event"; + info!(?event, "relay client event"); + } + + SwarmEvent::Behaviour(NodeEvent::RelayServer(event)) => { + event_string = "relay_server_event"; + + info!(?event, "relay server event"); + } SwarmEvent::Behaviour(NodeEvent::Identify(iden)) => { event_string = "identify"; @@ -344,6 +356,15 @@ impl SwarmDriver { return Ok(()); } + // check if this peer is a relay candidate that we had dialed. + if self.is_known_behind_nat { + self.relay_manager.try_update_on_connection_success( + &peer_id, + &info.protocols, + &mut self.swarm, + ); + } + let has_dialed = self.dialed_peers.contains(&peer_id); let peer_is_node = info.agent_version == IDENTIFY_NODE_VERSION_STR.to_string(); @@ -398,6 +419,13 @@ impl SwarmDriver { (true, None) }; + // add potential relay candidates. + self.relay_manager.add_potential_candidates( + &peer_id, + &addrs, + &info.protocols, + ); + // Add some autonat logic here for p in info.protocols { // TODO tie to an actual protocol version @@ -561,6 +589,12 @@ impl SwarmDriver { event_string = "OutgoingConnErr"; warn!("OutgoingConnectionError to {failed_peer_id:?} on {connection_id:?} - {error:?}"); + // try to update the relay manager on connection failure. + if self.is_known_behind_nat { + self.relay_manager + .try_update_on_connection_failure(&failed_peer_id); + } + // we need to decide if this was a critical error and the peer should be removed from the routing table let should_clean_peer = match error { DialError::Transport(errors) => { diff --git a/sn_networking/src/lib.rs b/sn_networking/src/lib.rs index 0851efd682..314750af54 100644 --- a/sn_networking/src/lib.rs +++ b/sn_networking/src/lib.rs @@ -24,6 +24,7 @@ mod metrics_service; mod network_discovery; mod record_store; mod record_store_api; +mod relay_manager; mod replication_fetcher; mod spends; pub mod target_arch; diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs new file mode 100644 index 0000000000..0d0d085892 --- /dev/null +++ b/sn_networking/src/relay_manager.rs @@ -0,0 +1,172 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::driver::NodeBehaviour; +use libp2p::{ + multiaddr::Protocol, + swarm::dial_opts::{DialOpts, PeerCondition}, + Multiaddr, PeerId, StreamProtocol, Swarm, +}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; + +const MAX_CONCURRENT_RELAY_CONNECTIONS: usize = 3; +const MAX_POTENTIAL_CANDIDATES: usize = 15; + +/// To manager relayed connections. +// todo: try to dial whenever connected_relays drops below threshold. Need to perform this on interval. +pub(crate) struct RelayManager { + connected_relays: BTreeMap, + dialing_relays: BTreeSet, + candidates: BTreeMap, +} + +impl RelayManager { + pub(crate) fn new(initial_peers: Vec) -> Self { + let candidates = initial_peers + .into_iter() + .filter_map(|addr| { + for protocol in addr.iter() { + if let Protocol::P2p(peer_id) = protocol { + return Some((peer_id, addr)); + } + } + None + }) + .collect(); + Self { + connected_relays: Default::default(), + dialing_relays: Default::default(), + candidates, + } + } + + pub(crate) fn add_potential_candidates( + &mut self, + peer_id: &PeerId, + addrs: &HashSet, + stream_protocols: &Vec, + ) { + if self.candidates.len() >= MAX_POTENTIAL_CANDIDATES { + return; + } + + if Self::does_it_support_relay_server_protocol(stream_protocols) { + // todo: collect and manage multiple addrs + if let Some(addr) = addrs.iter().next() { + // only consider non relayed peers + if !addr.iter().any(|p| p == Protocol::P2pCircuit) { + self.candidates.insert(*peer_id, addr.clone()); + } + } + } + } + + /// Dials candidate relays. + pub(crate) fn dial_relays(&mut self, swarm: &mut Swarm) { + let mut n_dialed = self.connected_relays.len(); + + if n_dialed >= MAX_CONCURRENT_RELAY_CONNECTIONS { + return; + } + + for (candidate_id, candidate_addr) in self.candidates.iter() { + match swarm.dial( + DialOpts::peer_id(candidate_id.clone()) + .condition(PeerCondition::NotDialing) + // todo: should we add P2pCircuit here? + // Just perform a direct connection and if the peer supports circuit protocol, the listen_on it. + // the `listen_on(adrr.with P2pP)` will create the persistent relay connection right? + .addresses(vec![candidate_addr.clone().with(Protocol::P2pCircuit)]) + .build(), + ) { + Ok(_) => { + info!("Dialing Relay: {candidate_id:?} succeeded."); + self.dialing_relays.insert(*candidate_id); + n_dialed += 1; + if n_dialed >= MAX_CONCURRENT_RELAY_CONNECTIONS { + return; + } + } + Err(err) => { + error!("Error while dialing relay: {candidate_id:?} {err:?}",); + } + } + } + } + + // todo: should we remove all our other `listen_addr`? Any should we block from adding `add_external_address` if + // we're behind nat? + pub(crate) fn try_update_on_connection_success( + &mut self, + peer_id: &PeerId, + stream_protocols: &Vec, + swarm: &mut Swarm, + ) { + if !self.dialing_relays.contains(peer_id) { + return; + } + + let _ = self.dialing_relays.remove(peer_id); + + // this can happen if the initial bootstrap peers does not support the protocol + if !Self::does_it_support_relay_server_protocol(stream_protocols) { + let _ = self.candidates.remove(peer_id); + error!("A dialed relay candidate does not support relay server protocol: {peer_id:?}"); + return; + } + + // todo: when should we clear out our previous non-relayed listen_addrs? + if let Some(addr) = self.candidates.remove(peer_id) { + // if we have less than threshold relay connections, listen on this relayed connection + if self.connected_relays.len() < MAX_CONCURRENT_RELAY_CONNECTIONS { + let relay_addr = addr.with(Protocol::P2pCircuit); + match swarm.listen_on(relay_addr.clone()) { + Ok(_) => { + info!("Relay connection established with {peer_id:?} on {relay_addr:?}"); + self.connected_relays.insert(*peer_id, relay_addr); + } + Err(err) => { + error!("Error while trying to listen on relayed connection: {err:?} on {relay_addr:?}"); + } + } + } + } else { + error!("Could not find relay candidate after successful connection: {peer_id:?}"); + } + } + + pub(crate) fn try_update_on_connection_failure(&mut self, peer_id: &PeerId) { + if !self.connected_relays.contains_key(peer_id) + && !self.dialing_relays.contains(peer_id) + && !self.candidates.contains_key(peer_id) + { + return; + } + + if let Some(addr) = self.connected_relays.remove(peer_id) { + debug!("Removing connected relay from {peer_id:?}: {addr:?} as we had a connection failure"); + } + + if self.dialing_relays.remove(peer_id) { + debug!("Removing dialing candidate {peer_id:?} as we had a connection failure"); + } + + if let Some(addr) = self.candidates.remove(peer_id) { + debug!("Removing relay candidate {peer_id:?}: {addr:?} as we had a connection failure"); + } + } + + fn does_it_support_relay_server_protocol(protocols: &Vec) -> bool { + for stream_protocol in protocols { + if *stream_protocol == "/libp2p/circuit/relay/0.2.0/stop" { + return true; + } + } + false + } +} diff --git a/sn_node/src/node.rs b/sn_node/src/node.rs index c3b979caf6..c19f6f9b13 100644 --- a/sn_node/src/node.rs +++ b/sn_node/src/node.rs @@ -134,6 +134,7 @@ impl NodeBuilder { network_builder.metrics_registry(metrics_registry); #[cfg(feature = "open-metrics")] network_builder.metrics_server_port(self.metrics_server_port); + network_builder.initial_peers(self.initial_peers.clone()); let (network, network_event_receiver, swarm_driver) = network_builder.build_node()?; let node_events_channel = NodeEventsChannel::default(); From 8505b522c1f51ea5070a158b6b2bc6908f950ba6 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 9 Apr 2024 18:43:30 +0530 Subject: [PATCH 098/205] chore: remove quic --- sn_networking/src/driver.rs | 10 ---------- sn_peers_acquisition/src/lib.rs | 4 +--- sn_protocol/src/lib.rs | 2 +- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index 89f189ea0f..732a23494e 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -315,15 +315,6 @@ impl NetworkBuilder { .listen_on(addr_tcp) .expect("Multiaddr should be supported by our configured transports"); - // Listen on QUIC - let addr_quic = Multiaddr::from(listen_socket_addr.ip()) - .with(Protocol::Udp(listen_socket_addr.port())) - .with(Protocol::QuicV1); - let _listener_id = swarm_driver - .swarm - .listen_on(addr_quic) - .expect("Multiaddr should be supported by our configured transports"); - // Listen on WebSocket #[cfg(any(feature = "websockets", target_arch = "wasm32"))] { @@ -475,7 +466,6 @@ impl NetworkBuilder { libp2p::yamux::Config::default, ) .map_err(|e| NetworkError::BahviourErr(e.to_string()))? - .with_quic() .with_relay_client(libp2p::noise::Config::new, libp2p::yamux::Config::default) .map_err(|e| NetworkError::BahviourErr(e.to_string()))? .with_behaviour(|_keypair, relay_behaviour| NodeBehaviour { diff --git a/sn_peers_acquisition/src/lib.rs b/sn_peers_acquisition/src/lib.rs index e1698ea374..a9d7035fa2 100644 --- a/sn_peers_acquisition/src/lib.rs +++ b/sn_peers_acquisition/src/lib.rs @@ -137,9 +137,7 @@ pub fn parse_peer_addr(addr: &str) -> Result { if let Ok(addr) = addr.parse::() { let start_addr = Multiaddr::from(*addr.ip()); // Start with an address into a `/ip4//udp//quic-v1` multiaddr. - let multiaddr = start_addr - .with(Protocol::Udp(addr.port())) - .with(Protocol::QuicV1); + let multiaddr = start_addr.with(Protocol::Tcp(addr.port())); #[cfg(all(feature = "websockets", feature = "wasm32"))] // Turn the address into a `/ip4//udp//websocket-websys-v1` multiaddr. diff --git a/sn_protocol/src/lib.rs b/sn_protocol/src/lib.rs index fc3657c825..7b93a05421 100644 --- a/sn_protocol/src/lib.rs +++ b/sn_protocol/src/lib.rs @@ -48,7 +48,7 @@ use xor_name::XorName; pub fn get_port_from_multiaddr(multi_addr: &Multiaddr) -> Option { // assuming the listening addr contains /ip4/127.0.0.1/udp/56215/quic-v1/p2p/ for protocol in multi_addr.iter() { - if let Protocol::Udp(port) = protocol { + if let Protocol::Tcp(port) = protocol { return Some(port); } } From 44f54f54ffed322d2132f472d7ee6b0b1b238593 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 9 Apr 2024 19:02:30 +0530 Subject: [PATCH 099/205] fix(relay_manager): do not dial with P2PCircuit protocol --- sn_networking/src/relay_manager.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index 0d0d085892..6cfe3dea31 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -76,12 +76,9 @@ impl RelayManager { for (candidate_id, candidate_addr) in self.candidates.iter() { match swarm.dial( - DialOpts::peer_id(candidate_id.clone()) + DialOpts::peer_id(*candidate_id) .condition(PeerCondition::NotDialing) - // todo: should we add P2pCircuit here? - // Just perform a direct connection and if the peer supports circuit protocol, the listen_on it. - // the `listen_on(adrr.with P2pP)` will create the persistent relay connection right? - .addresses(vec![candidate_addr.clone().with(Protocol::P2pCircuit)]) + .addresses(vec![candidate_addr.clone()]) .build(), ) { Ok(_) => { From f47abd3769aaf06d9de1de1bab07c4e704cf9187 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 9 Apr 2024 19:27:40 +0530 Subject: [PATCH 100/205] fix(network): do not perform AutoNat for clients --- sn_networking/src/event.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 65977786f0..f24290378d 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -261,6 +261,10 @@ impl SwarmDriver { SwarmEvent::Behaviour(NodeEvent::Autonat(autonat_event)) => { event_string = "autonat_event"; + if self.is_client { + return Ok(()); + } + match *autonat_event { libp2p::autonat::Event::StatusChanged { old, new } => { info!("NAT status change... was: {old:?} now: {new:?}"); From 7e15526ff3f58fe452395f1892c4b47f690d881a Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 9 Apr 2024 20:10:00 +0530 Subject: [PATCH 101/205] feat(relay): remove the dial flow --- sn_networking/src/driver.rs | 9 +++ sn_networking/src/event.rs | 26 ++----- sn_networking/src/relay_manager.rs | 120 +++++++++-------------------- 3 files changed, 53 insertions(+), 102 deletions(-) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index 732a23494e..ba188c957a 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -70,6 +70,9 @@ use tracing::warn; /// based upon our knowledge of the CLOSE_GROUP pub(crate) const CLOSET_RECORD_CHECK_INTERVAL: Duration = Duration::from_secs(15); +/// Interval over which we query relay manager to check if we can make any more reservations. +pub(crate) const RELAY_MANAGER_RESERVATION_INTERVAL: Duration = Duration::from_secs(30); + /// The ways in which the Get Closest queries are used. pub(crate) enum PendingGetClosestType { /// The network discovery method is present at the networking layer @@ -596,6 +599,7 @@ impl SwarmDriver { pub async fn run(mut self) { let mut bootstrap_interval = interval(BOOTSTRAP_INTERVAL); let mut set_farthest_record_interval = interval(CLOSET_RECORD_CHECK_INTERVAL); + let mut relay_manager_reservation_interval = interval(RELAY_MANAGER_RESERVATION_INTERVAL); loop { tokio::select! { @@ -637,6 +641,11 @@ impl SwarmDriver { } } } + _ = relay_manager_reservation_interval.tick() => { + if self.is_known_behind_nat { + self.relay_manager.try_connecting_to_relay(&mut self.swarm); + } + } } } } diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index f24290378d..d4a2f88269 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -273,9 +273,6 @@ impl SwarmDriver { if new == NatStatus::Private { self.is_known_behind_nat = true; warn!("BEHIND NAT>>>>>>>"); - // we are behind NAT - // do relay hole punch etc - self.relay_manager.dial_relays(&mut self.swarm); } else if let NatStatus::Public(addr) = new { self.is_known_behind_nat = false; info!("NOT BEHIND NAT go server mode!!"); @@ -323,6 +320,14 @@ impl SwarmDriver { event_string = "relay_client_event"; info!(?event, "relay client event"); + + if let libp2p::relay::client::Event::ReservationReqAccepted { + relay_peer_id, .. + } = *event + { + self.relay_manager + .update_on_successful_reservation(&relay_peer_id); + } } SwarmEvent::Behaviour(NodeEvent::RelayServer(event)) => { @@ -360,15 +365,6 @@ impl SwarmDriver { return Ok(()); } - // check if this peer is a relay candidate that we had dialed. - if self.is_known_behind_nat { - self.relay_manager.try_update_on_connection_success( - &peer_id, - &info.protocols, - &mut self.swarm, - ); - } - let has_dialed = self.dialed_peers.contains(&peer_id); let peer_is_node = info.agent_version == IDENTIFY_NODE_VERSION_STR.to_string(); @@ -593,12 +589,6 @@ impl SwarmDriver { event_string = "OutgoingConnErr"; warn!("OutgoingConnectionError to {failed_peer_id:?} on {connection_id:?} - {error:?}"); - // try to update the relay manager on connection failure. - if self.is_known_behind_nat { - self.relay_manager - .try_update_on_connection_failure(&failed_peer_id); - } - // we need to decide if this was a critical error and the peer should be removed from the routing table let should_clean_peer = match error { DialError::Transport(errors) => { diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index 6cfe3dea31..0c67996824 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -7,12 +7,8 @@ // permissions and limitations relating to use of the SAFE Network Software. use crate::driver::NodeBehaviour; -use libp2p::{ - multiaddr::Protocol, - swarm::dial_opts::{DialOpts, PeerCondition}, - Multiaddr, PeerId, StreamProtocol, Swarm, -}; -use std::collections::{BTreeMap, BTreeSet, HashSet}; +use libp2p::{multiaddr::Protocol, Multiaddr, PeerId, StreamProtocol, Swarm}; +use std::collections::{BTreeMap, HashSet, VecDeque}; const MAX_CONCURRENT_RELAY_CONNECTIONS: usize = 3; const MAX_POTENTIAL_CANDIDATES: usize = 15; @@ -21,8 +17,8 @@ const MAX_POTENTIAL_CANDIDATES: usize = 15; // todo: try to dial whenever connected_relays drops below threshold. Need to perform this on interval. pub(crate) struct RelayManager { connected_relays: BTreeMap, - dialing_relays: BTreeSet, - candidates: BTreeMap, + waiting_for_reservation: BTreeMap, + candidates: VecDeque<(PeerId, Multiaddr)>, } impl RelayManager { @@ -40,11 +36,13 @@ impl RelayManager { .collect(); Self { connected_relays: Default::default(), - dialing_relays: Default::default(), + waiting_for_reservation: Default::default(), candidates, } } + /// Add a potential candidate to the list if it satisfies all the identify checks and also supports the relay server + /// protocol. pub(crate) fn add_potential_candidates( &mut self, peer_id: &PeerId, @@ -60,101 +58,55 @@ impl RelayManager { if let Some(addr) = addrs.iter().next() { // only consider non relayed peers if !addr.iter().any(|p| p == Protocol::P2pCircuit) { - self.candidates.insert(*peer_id, addr.clone()); + self.candidates.push_back((*peer_id, addr.clone())); } } } } - /// Dials candidate relays. - pub(crate) fn dial_relays(&mut self, swarm: &mut Swarm) { - let mut n_dialed = self.connected_relays.len(); - - if n_dialed >= MAX_CONCURRENT_RELAY_CONNECTIONS { + // todo: how do we know if a reservation has been revoked / if the peer has gone offline? + /// Try connecting to candidate relays if we are below the threshold connections. + /// This is run periodically on a loop. + pub(crate) fn try_connecting_to_relay(&mut self, swarm: &mut Swarm) { + if self.connected_relays.len() >= MAX_CONCURRENT_RELAY_CONNECTIONS { return; } - for (candidate_id, candidate_addr) in self.candidates.iter() { - match swarm.dial( - DialOpts::peer_id(*candidate_id) - .condition(PeerCondition::NotDialing) - .addresses(vec![candidate_addr.clone()]) - .build(), - ) { - Ok(_) => { - info!("Dialing Relay: {candidate_id:?} succeeded."); - self.dialing_relays.insert(*candidate_id); - n_dialed += 1; - if n_dialed >= MAX_CONCURRENT_RELAY_CONNECTIONS { - return; - } - } - Err(err) => { - error!("Error while dialing relay: {candidate_id:?} {err:?}",); - } - } - } - } - - // todo: should we remove all our other `listen_addr`? Any should we block from adding `add_external_address` if - // we're behind nat? - pub(crate) fn try_update_on_connection_success( - &mut self, - peer_id: &PeerId, - stream_protocols: &Vec, - swarm: &mut Swarm, - ) { - if !self.dialing_relays.contains(peer_id) { - return; - } + let reservations_to_make = MAX_CONCURRENT_RELAY_CONNECTIONS - self.connected_relays.len(); + let mut n_reservations = 0; - let _ = self.dialing_relays.remove(peer_id); - - // this can happen if the initial bootstrap peers does not support the protocol - if !Self::does_it_support_relay_server_protocol(stream_protocols) { - let _ = self.candidates.remove(peer_id); - error!("A dialed relay candidate does not support relay server protocol: {peer_id:?}"); - return; - } - - // todo: when should we clear out our previous non-relayed listen_addrs? - if let Some(addr) = self.candidates.remove(peer_id) { - // if we have less than threshold relay connections, listen on this relayed connection - if self.connected_relays.len() < MAX_CONCURRENT_RELAY_CONNECTIONS { + while n_reservations < reservations_to_make { + // todo: should we remove all our other `listen_addr`? And should we block from adding `add_external_address` if + // we're behind nat? + if let Some((peer_id, addr)) = self.candidates.pop_front() { let relay_addr = addr.with(Protocol::P2pCircuit); match swarm.listen_on(relay_addr.clone()) { Ok(_) => { - info!("Relay connection established with {peer_id:?} on {relay_addr:?}"); - self.connected_relays.insert(*peer_id, relay_addr); + info!("Sending reservation to relay {peer_id:?} on {relay_addr:?}"); + self.waiting_for_reservation.insert(peer_id, relay_addr); + n_reservations += 1; } Err(err) => { - error!("Error while trying to listen on relayed connection: {err:?} on {relay_addr:?}"); + error!("Error while trying to listen on the relay addr: {err:?} on {relay_addr:?}"); } } + } else { + error!("No more relay candidates"); + break; } - } else { - error!("Could not find relay candidate after successful connection: {peer_id:?}"); } } - pub(crate) fn try_update_on_connection_failure(&mut self, peer_id: &PeerId) { - if !self.connected_relays.contains_key(peer_id) - && !self.dialing_relays.contains(peer_id) - && !self.candidates.contains_key(peer_id) - { - return; - } - - if let Some(addr) = self.connected_relays.remove(peer_id) { - debug!("Removing connected relay from {peer_id:?}: {addr:?} as we had a connection failure"); - } - - if self.dialing_relays.remove(peer_id) { - debug!("Removing dialing candidate {peer_id:?} as we had a connection failure"); - } - - if let Some(addr) = self.candidates.remove(peer_id) { - debug!("Removing relay candidate {peer_id:?}: {addr:?} as we had a connection failure"); + /// Update our state after we've successfully made reservation with a relay. + pub(crate) fn update_on_successful_reservation(&mut self, peer_id: &PeerId) { + match self.waiting_for_reservation.remove(peer_id) { + Some(addr) => { + info!("Successfully made reservation with {peer_id:?} on {addr:?}"); + self.connected_relays.insert(*peer_id, addr); + } + None => { + debug!("Made a reservation with a peer that we had not requested to"); + } } } From 0276f48bf1ba26bd811f503492155890c136cc79 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 9 Apr 2024 20:48:23 +0530 Subject: [PATCH 102/205] feat(relay): update the relay manager if the listen addr has been closed --- sn_networking/src/event.rs | 9 ++++++++ sn_networking/src/relay_manager.rs | 33 +++++++++++++++++++++++++----- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index d4a2f88269..561efabebd 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -540,6 +540,15 @@ impl SwarmDriver { info!("Local node is listening on {address:?}"); } + SwarmEvent::ListenerClosed { + listener_id, + addresses, + reason, + } => { + event_string = "listener closed"; + info!("Listener {listener_id:?} with add {addresses:?} has been closed for {reason:?}"); + self.relay_manager.update_on_listener_closed(&listener_id); + } SwarmEvent::IncomingConnection { connection_id, local_addr, diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index 0c67996824..13a9000361 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -7,8 +7,10 @@ // permissions and limitations relating to use of the SAFE Network Software. use crate::driver::NodeBehaviour; -use libp2p::{multiaddr::Protocol, Multiaddr, PeerId, StreamProtocol, Swarm}; -use std::collections::{BTreeMap, HashSet, VecDeque}; +use libp2p::{ + core::transport::ListenerId, multiaddr::Protocol, Multiaddr, PeerId, StreamProtocol, Swarm, +}; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; const MAX_CONCURRENT_RELAY_CONNECTIONS: usize = 3; const MAX_POTENTIAL_CANDIDATES: usize = 15; @@ -16,9 +18,13 @@ const MAX_POTENTIAL_CANDIDATES: usize = 15; /// To manager relayed connections. // todo: try to dial whenever connected_relays drops below threshold. Need to perform this on interval. pub(crate) struct RelayManager { - connected_relays: BTreeMap, - waiting_for_reservation: BTreeMap, + // states candidates: VecDeque<(PeerId, Multiaddr)>, + waiting_for_reservation: BTreeMap, + connected_relays: BTreeMap, + + // misc + listener_id_map: HashMap, } impl RelayManager { @@ -38,6 +44,7 @@ impl RelayManager { connected_relays: Default::default(), waiting_for_reservation: Default::default(), candidates, + listener_id_map: Default::default(), } } @@ -81,9 +88,10 @@ impl RelayManager { if let Some((peer_id, addr)) = self.candidates.pop_front() { let relay_addr = addr.with(Protocol::P2pCircuit); match swarm.listen_on(relay_addr.clone()) { - Ok(_) => { + Ok(id) => { info!("Sending reservation to relay {peer_id:?} on {relay_addr:?}"); self.waiting_for_reservation.insert(peer_id, relay_addr); + self.listener_id_map.insert(id, peer_id); n_reservations += 1; } Err(err) => { @@ -110,6 +118,21 @@ impl RelayManager { } } + /// Update our state if the reservation has been cancelled or if the relay has closed. + pub(crate) fn update_on_listener_closed(&mut self, listener_id: &ListenerId) { + let Some(peer_id) = self.listener_id_map.remove(listener_id) else { + return; + }; + + if let Some(addr) = self.connected_relays.remove(&peer_id) { + info!("Removed peer form connected_relays as the listener has been closed {peer_id:?}: {addr:?}"); + } else if let Some(addr) = self.waiting_for_reservation.remove(&peer_id) { + info!("Removed peer form waiting_for_reservation as the listener has been closed {peer_id:?}: {addr:?}"); + } else { + warn!("Could not find the listen addr after making reservation to the same"); + } + } + fn does_it_support_relay_server_protocol(protocols: &Vec) -> bool { for stream_protocol in protocols { if *stream_protocol == "/libp2p/circuit/relay/0.2.0/stop" { From ba5574d9eb3a088c84d62d8a96042d235d43e8a3 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 9 Apr 2024 22:56:57 +0530 Subject: [PATCH 103/205] chore(relay): add candidate even if we are dialing --- sn_networking/src/event.rs | 18 ++++++++++-------- sn_networking/src/relay_manager.rs | 7 +++++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 561efabebd..b8fff5b72d 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -272,7 +272,7 @@ impl SwarmDriver { // if we are private, establish relay if new == NatStatus::Private { self.is_known_behind_nat = true; - warn!("BEHIND NAT>>>>>>>"); + info!("BEHIND NAT"); } else if let NatStatus::Public(addr) = new { self.is_known_behind_nat = false; info!("NOT BEHIND NAT go server mode!!"); @@ -386,6 +386,15 @@ impl SwarmDriver { .collect(), }; + if !self.local && peer_is_node { + // add potential relay candidates. + self.relay_manager.add_potential_candidates( + &peer_id, + &addrs, + &info.protocols, + ); + } + // When received an identify from un-dialed peer, try to dial it // The dial shall trigger the same identify to be sent again and confirm // peer is external accessible, hence safe to be added into RT. @@ -419,13 +428,6 @@ impl SwarmDriver { (true, None) }; - // add potential relay candidates. - self.relay_manager.add_potential_candidates( - &peer_id, - &addrs, - &info.protocols, - ); - // Add some autonat logic here for p in info.protocols { // TODO tie to an actual protocol version diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index 13a9000361..60a1c65f07 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -65,6 +65,7 @@ impl RelayManager { if let Some(addr) = addrs.iter().next() { // only consider non relayed peers if !addr.iter().any(|p| p == Protocol::P2pCircuit) { + debug!("Adding {peer_id:?} with {addr:?} as a potential relay candidate"); self.candidates.push_back((*peer_id, addr.clone())); } } @@ -75,7 +76,9 @@ impl RelayManager { /// Try connecting to candidate relays if we are below the threshold connections. /// This is run periodically on a loop. pub(crate) fn try_connecting_to_relay(&mut self, swarm: &mut Swarm) { - if self.connected_relays.len() >= MAX_CONCURRENT_RELAY_CONNECTIONS { + if self.connected_relays.len() >= MAX_CONCURRENT_RELAY_CONNECTIONS + || self.candidates.is_empty() + { return; } @@ -99,7 +102,7 @@ impl RelayManager { } } } else { - error!("No more relay candidates"); + debug!("No more relay candidates."); break; } } From e80e138a0bb74488789b106f2803c052ccd09919 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 9 Apr 2024 23:48:15 +0530 Subject: [PATCH 104/205] feat(relay): remove old listen addr if we are using a relayed connection --- sn_networking/src/cmd.rs | 15 --------- sn_networking/src/driver.rs | 24 +++++++-------- sn_networking/src/event.rs | 5 ++- sn_networking/src/relay_manager.rs | 49 +++++++++++++++++++++++++----- 4 files changed, 55 insertions(+), 38 deletions(-) diff --git a/sn_networking/src/cmd.rs b/sn_networking/src/cmd.rs index 8729338a8e..8215741b70 100644 --- a/sn_networking/src/cmd.rs +++ b/sn_networking/src/cmd.rs @@ -54,10 +54,6 @@ pub enum NodeIssue { /// Commands to send to the Swarm #[allow(clippy::large_enum_variant)] pub enum SwarmCmd { - StartListening { - addr: Multiaddr, - sender: oneshot::Sender>, - }, Dial { addr: Multiaddr, sender: oneshot::Sender>, @@ -189,9 +185,6 @@ pub enum SwarmCmd { impl Debug for SwarmCmd { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - SwarmCmd::StartListening { addr, .. } => { - write!(f, "SwarmCmd::StartListening {{ addr: {addr:?} }}") - } SwarmCmd::Dial { addr, .. } => { write!(f, "SwarmCmd::Dial {{ addr: {addr:?} }}") } @@ -577,14 +570,6 @@ impl SwarmDriver { .record_addresses(); let _ = sender.send(addresses); } - - SwarmCmd::StartListening { addr, sender } => { - cmd_string = "StartListening"; - let _ = match self.swarm.listen_on(addr) { - Ok(_) => sender.send(Ok(())), - Err(e) => sender.send(Err(e.into())), - }; - } SwarmCmd::Dial { addr, sender } => { cmd_string = "Dial"; diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index ba188c957a..c1a85f61b4 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -312,9 +312,7 @@ impl NetworkBuilder { // Listen on TCP for autonat let addr_tcp = Multiaddr::from(listen_socket_addr.ip()).with(Protocol::Tcp(listen_socket_addr.port())); - - let _listener_id = swarm_driver - .swarm + swarm_driver .listen_on(addr_tcp) .expect("Multiaddr should be supported by our configured transports"); @@ -324,8 +322,7 @@ impl NetworkBuilder { let addr_ws = Multiaddr::from(listen_socket_addr.ip()) .with(Protocol::Tcp(listen_socket_addr.port())) .with(Protocol::Ws("/".into())); - let _listener_id = swarm_driver - .swarm + swarm_driver .listen_on(addr_ws) .expect("Multiaddr should be supported by our configured transports"); } @@ -501,7 +498,6 @@ impl NetworkBuilder { local: self.local, listen_port: self.listen_addr.map(|addr| addr.port()), is_client, - is_known_behind_nat: false, connected_peers: 0, bootstrap, relay_manager: RelayManager::new(self.initial_peers), @@ -558,8 +554,6 @@ pub struct SwarmDriver { #[allow(unused)] pub(crate) network_metrics: NetworkMetrics, - pub(crate) is_known_behind_nat: bool, - cmd_receiver: mpsc::Receiver, event_sender: mpsc::Sender, // Use `self.send_event()` to send a NetworkEvent. @@ -641,11 +635,7 @@ impl SwarmDriver { } } } - _ = relay_manager_reservation_interval.tick() => { - if self.is_known_behind_nat { - self.relay_manager.try_connecting_to_relay(&mut self.swarm); - } - } + _ = relay_manager_reservation_interval.tick() => self.relay_manager.try_connecting_to_relay(&mut self.swarm), } } } @@ -797,4 +787,12 @@ impl SwarmDriver { self.handling_statistics.clear(); } } + + /// Listen on the provided address. Also records it within RelayManager + pub(crate) fn listen_on(&mut self, addr: Multiaddr) -> Result<()> { + let id = self.swarm.listen_on(addr)?; + self.relay_manager.add_non_relayed_listener_id(id); + + Ok(()) + } } diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index b8fff5b72d..8b79014dc1 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -270,11 +270,10 @@ impl SwarmDriver { info!("NAT status change... was: {old:?} now: {new:?}"); // if we are private, establish relay + self.relay_manager.set_nat_status(&new); if new == NatStatus::Private { - self.is_known_behind_nat = true; info!("BEHIND NAT"); } else if let NatStatus::Public(addr) = new { - self.is_known_behind_nat = false; info!("NOT BEHIND NAT go server mode!!"); // Trigger server mode if we're not a client @@ -326,7 +325,7 @@ impl SwarmDriver { } = *event { self.relay_manager - .update_on_successful_reservation(&relay_peer_id); + .update_on_successful_reservation(&relay_peer_id, &mut self.swarm); } } diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index 60a1c65f07..f64478146e 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -8,7 +8,8 @@ use crate::driver::NodeBehaviour; use libp2p::{ - core::transport::ListenerId, multiaddr::Protocol, Multiaddr, PeerId, StreamProtocol, Swarm, + autonat::NatStatus, core::transport::ListenerId, multiaddr::Protocol, Multiaddr, PeerId, + StreamProtocol, Swarm, }; use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; @@ -19,12 +20,16 @@ const MAX_POTENTIAL_CANDIDATES: usize = 15; // todo: try to dial whenever connected_relays drops below threshold. Need to perform this on interval. pub(crate) struct RelayManager { // states + we_are_behind_nat: bool, candidates: VecDeque<(PeerId, Multiaddr)>, waiting_for_reservation: BTreeMap, connected_relays: BTreeMap, - // misc - listener_id_map: HashMap, + /// Tracker for the relayed listen addresses. + relayed_listener_id_map: HashMap, + /// Tracker for the non relayed listen addresses. These should be collected whenever we call `listen_on` from outside + /// the manager. + non_relayed_listener_id: VecDeque, } impl RelayManager { @@ -41,13 +46,27 @@ impl RelayManager { }) .collect(); Self { + we_are_behind_nat: false, connected_relays: Default::default(), waiting_for_reservation: Default::default(), candidates, - listener_id_map: Default::default(), + non_relayed_listener_id: Default::default(), + relayed_listener_id_map: Default::default(), } } + pub(crate) fn set_nat_status(&mut self, nat_status: &NatStatus) { + match nat_status { + NatStatus::Private => self.we_are_behind_nat = true, + _ => self.we_are_behind_nat = false, + } + } + + pub(crate) fn add_non_relayed_listener_id(&mut self, listener_id: ListenerId) { + debug!("Adding non relayed listener id: {listener_id:?}"); + self.non_relayed_listener_id.push_front(listener_id); + } + /// Add a potential candidate to the list if it satisfies all the identify checks and also supports the relay server /// protocol. pub(crate) fn add_potential_candidates( @@ -76,6 +95,10 @@ impl RelayManager { /// Try connecting to candidate relays if we are below the threshold connections. /// This is run periodically on a loop. pub(crate) fn try_connecting_to_relay(&mut self, swarm: &mut Swarm) { + if !self.we_are_behind_nat { + return; + } + if self.connected_relays.len() >= MAX_CONCURRENT_RELAY_CONNECTIONS || self.candidates.is_empty() { @@ -94,7 +117,7 @@ impl RelayManager { Ok(id) => { info!("Sending reservation to relay {peer_id:?} on {relay_addr:?}"); self.waiting_for_reservation.insert(peer_id, relay_addr); - self.listener_id_map.insert(id, peer_id); + self.relayed_listener_id_map.insert(id, peer_id); n_reservations += 1; } Err(err) => { @@ -109,7 +132,19 @@ impl RelayManager { } /// Update our state after we've successfully made reservation with a relay. - pub(crate) fn update_on_successful_reservation(&mut self, peer_id: &PeerId) { + pub(crate) fn update_on_successful_reservation( + &mut self, + peer_id: &PeerId, + swarm: &mut Swarm, + ) { + // now that we have made a reservation, remove our non-relayed listeners + while !self.non_relayed_listener_id.is_empty() { + if let Some(listener_id) = self.non_relayed_listener_id.pop_back() { + let res = swarm.remove_listener(listener_id); + debug!("Removed {listener_id:?} with result: {res} from swarm as we now have a relay reservation"); + } + } + match self.waiting_for_reservation.remove(peer_id) { Some(addr) => { info!("Successfully made reservation with {peer_id:?} on {addr:?}"); @@ -123,7 +158,7 @@ impl RelayManager { /// Update our state if the reservation has been cancelled or if the relay has closed. pub(crate) fn update_on_listener_closed(&mut self, listener_id: &ListenerId) { - let Some(peer_id) = self.listener_id_map.remove(listener_id) else { + let Some(peer_id) = self.relayed_listener_id_map.remove(listener_id) else { return; }; From 764b2c6c9575d26647bc40f2d8ab1d0825922f20 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 10 Apr 2024 13:04:49 +0530 Subject: [PATCH 105/205] chore: log listner id --- sn_networking/src/event.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 8b79014dc1..ad576ac643 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -520,7 +520,10 @@ impl SwarmDriver { } } - SwarmEvent::NewListenAddr { address, .. } => { + SwarmEvent::NewListenAddr { + address, + listener_id, + } => { event_string = "new listen addr"; // update our stored port if it is configured to be 0 or None @@ -539,7 +542,7 @@ impl SwarmDriver { self.send_event(NetworkEvent::NewListenAddr(address.clone())); - info!("Local node is listening on {address:?}"); + info!("Local node is listening (id: {listener_id}) on {address:?}"); } SwarmEvent::ListenerClosed { listener_id, From 554d7cf0cc729091ae30fe9e61af35a6f73a2527 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 10 Apr 2024 18:04:26 +0530 Subject: [PATCH 106/205] feat(relay): remove autonat and enable hole punching manually --- Cargo.lock | 22 ------ sn_networking/Cargo.toml | 1 - sn_networking/src/cmd.rs | 5 +- sn_networking/src/driver.rs | 21 ++--- sn_networking/src/event.rs | 119 ++++++++--------------------- sn_networking/src/relay_manager.rs | 42 +++++++--- sn_node/src/bin/safenode/main.rs | 14 +++- sn_node/src/event.rs | 2 +- sn_node/src/node.rs | 11 ++- 9 files changed, 98 insertions(+), 139 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1cbfe514e9..97f9f2e5f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2671,7 +2671,6 @@ dependencies = [ "getrandom", "instant", "libp2p-allow-block-list", - "libp2p-autonat", "libp2p-connection-limits", "libp2p-core", "libp2p-dcutr", @@ -2710,27 +2709,6 @@ dependencies = [ "void", ] -[[package]] -name = "libp2p-autonat" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95151726170e41b591735bf95c42b888fe4aa14f65216a9fbf0edcc04510586" -dependencies = [ - "async-trait", - "asynchronous-codec 0.6.2", - "futures", - "futures-timer", - "instant", - "libp2p-core", - "libp2p-identity", - "libp2p-request-response", - "libp2p-swarm", - "quick-protobuf", - "quick-protobuf-codec 0.2.0", - "rand", - "tracing", -] - [[package]] name = "libp2p-connection-limits" version = "0.3.1" diff --git a/sn_networking/Cargo.toml b/sn_networking/Cargo.toml index 41fe226950..fc0fb04c98 100644 --- a/sn_networking/Cargo.toml +++ b/sn_networking/Cargo.toml @@ -24,7 +24,6 @@ libp2p = { version = "0.53", features = [ "tokio", "dns", "kad", - "autonat", "macros", "request-response", "cbor", diff --git a/sn_networking/src/cmd.rs b/sn_networking/src/cmd.rs index 8215741b70..edb6974cc6 100644 --- a/sn_networking/src/cmd.rs +++ b/sn_networking/src/cmd.rs @@ -9,6 +9,7 @@ use crate::{ driver::{PendingGetClosestType, SwarmDriver}, error::{NetworkError, Result}, + event::TerminateNodeReason, multiaddr_pop_p2p, GetRecordCfg, GetRecordError, MsgResponder, NetworkEvent, CLOSE_GROUP_SIZE, REPLICATION_PEERS_COUNT, }; @@ -546,7 +547,9 @@ impl SwarmDriver { // When there is certain amount of continuous HDD write error, // the hard disk is considered as full, and the node shall be terminated. if self.hard_disk_write_error > MAX_CONTINUOUS_HDD_WRITE_ERROR { - self.send_event(NetworkEvent::TerminateNode); + self.send_event(NetworkEvent::TerminateNode { + reason: TerminateNodeReason::HardDiskWriteError, + }); } } SwarmCmd::RecordStoreHasKey { key, sender } => { diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index c1a85f61b4..1ec7339078 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -31,7 +31,6 @@ use futures::StreamExt; #[cfg(feature = "local-discovery")] use libp2p::mdns; use libp2p::{ - autonat, identity::Keypair, kad::{self, QueryId, Quorum, Record, K_VALUE}, multiaddr::Protocol, @@ -174,7 +173,6 @@ pub(super) struct NodeBehaviour { #[cfg(feature = "local-discovery")] pub(super) mdns: mdns::tokio::Behaviour, pub(super) identify: libp2p::identify::Behaviour, - pub(super) autonat: libp2p::autonat::Behaviour, pub(super) dcutr: libp2p::dcutr::Behaviour, pub(super) relay_client: libp2p::relay::client::Behaviour, pub(super) relay_server: libp2p::relay::Behaviour, @@ -182,6 +180,7 @@ pub(super) struct NodeBehaviour { #[derive(Debug)] pub struct NetworkBuilder { + enable_hole_punching: bool, keypair: Keypair, local: bool, root_dir: PathBuf, @@ -198,6 +197,7 @@ pub struct NetworkBuilder { impl NetworkBuilder { pub fn new(keypair: Keypair, local: bool, root_dir: PathBuf) -> Self { Self { + enable_hole_punching: false, keypair, local, root_dir, @@ -212,6 +212,10 @@ impl NetworkBuilder { } } + pub fn enable_hole_punching(&mut self, enable: bool) { + self.enable_hole_punching = enable; + } + pub fn listen_addr(&mut self, listen_addr: SocketAddr) { self.listen_addr = Some(listen_addr); } @@ -474,13 +478,6 @@ impl NetworkBuilder { request_response, #[cfg(feature = "local-discovery")] mdns, - autonat: autonat::Behaviour::new( - peer_id, - autonat::Config { - only_global_ips: false, - ..Default::default() - }, - ), identify, kademlia, dcutr: libp2p::dcutr::Behaviour::new(peer_id), @@ -491,6 +488,10 @@ impl NetworkBuilder { let bootstrap = ContinuousBootstrap::new(); let replication_fetcher = ReplicationFetcher::new(peer_id, network_event_sender.clone()); + let mut relay_manager = RelayManager::new(self.initial_peers); + if !is_client { + relay_manager.enable_hole_punching(self.enable_hole_punching); + } let swarm_driver = SwarmDriver { swarm, @@ -500,7 +501,7 @@ impl NetworkBuilder { is_client, connected_peers: 0, bootstrap, - relay_manager: RelayManager::new(self.initial_peers), + relay_manager, close_group: Default::default(), replication_fetcher, #[cfg(feature = "open-metrics")] diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index ad576ac643..573138dc8a 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -20,7 +20,6 @@ use itertools::Itertools; #[cfg(feature = "local-discovery")] use libp2p::mdns; use libp2p::{ - autonat::NatStatus, kad::{self, GetClosestPeersError, InboundRequest, QueryResult, Record, RecordKey, K_VALUE}, multiaddr::Protocol, request_response::{self, Message, ResponseChannel as PeerResponseChannel}, @@ -55,7 +54,6 @@ pub(super) enum NodeEvent { #[cfg(feature = "local-discovery")] Mdns(Box), Identify(Box), - Autonat(Box), Dcutr(Box), RelayClient(Box), RelayServer(Box), @@ -85,11 +83,6 @@ impl From for NodeEvent { NodeEvent::Identify(Box::new(event)) } } -impl From for NodeEvent { - fn from(event: libp2p::autonat::Event) -> Self { - NodeEvent::Autonat(Box::new(event)) - } -} impl From for NodeEvent { fn from(event: libp2p::dcutr::Event) -> Self { NodeEvent::Dcutr(Box::new(event)) @@ -151,8 +144,8 @@ pub enum NetworkEvent { NewListenAddr(Multiaddr), /// Report unverified record UnverifiedRecord(Record), - /// Terminate Node on HDD write erros - TerminateNode, + /// Terminate Node on unrecoverable errors + TerminateNode { reason: TerminateNodeReason }, /// List of peer nodes that failed to fetch replication copy from. FailedToFetchHolders(BTreeSet), /// A peer in RT that supposed to be verified. @@ -166,6 +159,13 @@ pub enum NetworkEvent { }, } +/// Terminate node for the following reason +#[derive(Debug, Clone)] +pub enum TerminateNodeReason { + HardDiskWriteError, + BehindNAT, +} + // Manually implement Debug as `#[debug(with = "unverified_record_fmt")]` not working as expected. impl Debug for NetworkEvent { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { @@ -212,8 +212,8 @@ impl Debug for NetworkEvent { let pretty_key = PrettyPrintRecordKey::from(&record.key); write!(f, "NetworkEvent::UnverifiedRecord({pretty_key:?})") } - NetworkEvent::TerminateNode => { - write!(f, "NetworkEvent::TerminateNode") + NetworkEvent::TerminateNode { reason } => { + write!(f, "NetworkEvent::TerminateNode({reason:?})") } NetworkEvent::FailedToFetchHolders(bad_nodes) => { write!(f, "NetworkEvent::FailedToFetchHolders({bad_nodes:?})") @@ -257,64 +257,6 @@ impl SwarmDriver { event_string = "kad_event"; self.handle_kad_event(kad_event)?; } - // Handle the autonat events from the libp2p swarm. - SwarmEvent::Behaviour(NodeEvent::Autonat(autonat_event)) => { - event_string = "autonat_event"; - - if self.is_client { - return Ok(()); - } - - match *autonat_event { - libp2p::autonat::Event::StatusChanged { old, new } => { - info!("NAT status change... was: {old:?} now: {new:?}"); - - // if we are private, establish relay - self.relay_manager.set_nat_status(&new); - if new == NatStatus::Private { - info!("BEHIND NAT"); - } else if let NatStatus::Public(addr) = new { - info!("NOT BEHIND NAT go server mode!!"); - - // Trigger server mode if we're not a client - if !self.is_client { - if self.local { - // all addresses are effectively external here... - // this is needed for Kad Mode::Server - self.swarm.add_external_address(addr); - } else { - // only add our global addresses - if multiaddr_is_global(&addr) { - self.swarm.add_external_address(addr); - } - } - } - } - } - libp2p::autonat::Event::InboundProbe(in_event) => { - debug!("NAT inbound probe"); - } - libp2p::autonat::Event::OutboundProbe(out_event) => match out_event { - libp2p::autonat::OutboundProbeEvent::Error { - probe_id, - peer, - error, - } => { - error!(?peer, ?error, "NAT outbound probe failed"); - } - libp2p::autonat::OutboundProbeEvent::Response { - probe_id, - peer, - address, - } => { - info!(%peer, ?address, "NAT outbound probe success"); - } - libp2p::autonat::OutboundProbeEvent::Request { probe_id, peer } => { - info!(%peer, "NAT outbound probe request"); - } - }, - } - } SwarmEvent::Behaviour(NodeEvent::RelayClient(event)) => { event_string = "relay_client_event"; @@ -427,25 +369,6 @@ impl SwarmDriver { (true, None) }; - // Add some autonat logic here - for p in info.protocols { - // TODO tie to an actual protocol version - if p.to_string().contains("autonat") { - info!(%peer_id, ?addrs, "peer supports autonat: {:?}",peer_id); - - for a in &addrs { - // add each address to the autonat server list - self.swarm - .behaviour_mut() - .autonat - .add_server(peer_id, Some(a.clone())); - } - - // break out - break; - } - } - if !kbucket_full { info!(%peer_id, ?addrs, "received identify info from undialed peer for not full kbucket {ilog2:?}, dial back to confirm external accessible"); self.dialed_peers.push(peer_id); @@ -487,6 +410,12 @@ impl SwarmDriver { .kademlia .add_address(&peer_id, multiaddr.clone()); } + + if self.relay_manager.are_we_behind_nat(&mut self.swarm) { + self.send_event(NetworkEvent::TerminateNode { + reason: TerminateNodeReason::BehindNAT, + }) + } } } } @@ -540,6 +469,20 @@ impl SwarmDriver { let local_peer_id = *self.swarm.local_peer_id(); let address = address.with(Protocol::P2p(local_peer_id)); + // Trigger server mode if we're not a client + if !self.is_client { + if self.local { + // all addresses are effectively external here... + // this is needed for Kad Mode::Server + self.swarm.add_external_address(address.clone()); + } else { + // only add our global addresses + if multiaddr_is_global(&address) { + self.swarm.add_external_address(address.clone()); + } + } + } + self.send_event(NetworkEvent::NewListenAddr(address.clone())); info!("Local node is listening (id: {listener_id}) on {address:?}"); diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index f64478146e..ec4f5a11fc 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -8,19 +8,19 @@ use crate::driver::NodeBehaviour; use libp2p::{ - autonat::NatStatus, core::transport::ListenerId, multiaddr::Protocol, Multiaddr, PeerId, - StreamProtocol, Swarm, + core::transport::ListenerId, multiaddr::Protocol, Multiaddr, PeerId, StreamProtocol, Swarm, }; use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; const MAX_CONCURRENT_RELAY_CONNECTIONS: usize = 3; const MAX_POTENTIAL_CANDIDATES: usize = 15; +const MAX_PEERS_IN_RT_DURING_NAT_CHECK: usize = 30; /// To manager relayed connections. // todo: try to dial whenever connected_relays drops below threshold. Need to perform this on interval. pub(crate) struct RelayManager { // states - we_are_behind_nat: bool, + enabled: bool, candidates: VecDeque<(PeerId, Multiaddr)>, waiting_for_reservation: BTreeMap, connected_relays: BTreeMap, @@ -46,7 +46,7 @@ impl RelayManager { }) .collect(); Self { - we_are_behind_nat: false, + enabled: false, connected_relays: Default::default(), waiting_for_reservation: Default::default(), candidates, @@ -55,11 +55,9 @@ impl RelayManager { } } - pub(crate) fn set_nat_status(&mut self, nat_status: &NatStatus) { - match nat_status { - NatStatus::Private => self.we_are_behind_nat = true, - _ => self.we_are_behind_nat = false, - } + pub(crate) fn enable_hole_punching(&mut self, enable: bool) { + info!("Setting enable hole punching to {enable:?}"); + self.enabled = enable; } pub(crate) fn add_non_relayed_listener_id(&mut self, listener_id: ListenerId) { @@ -67,6 +65,30 @@ impl RelayManager { self.non_relayed_listener_id.push_front(listener_id); } + /// If we have 0 incoming connection even after we have a lot of peers, then we are behind a NAT + pub(crate) fn are_we_behind_nat(&self, swarm: &mut Swarm) -> bool { + if swarm + .network_info() + .connection_counters() + .num_established_incoming() + == 0 + || swarm + .network_info() + .connection_counters() + .num_pending_incoming() + == 0 + { + let mut total_peers = 0; + for kbucket in swarm.behaviour_mut().kademlia.kbuckets() { + total_peers += kbucket.num_entries(); + if total_peers > MAX_PEERS_IN_RT_DURING_NAT_CHECK { + return true; + } + } + } + false + } + /// Add a potential candidate to the list if it satisfies all the identify checks and also supports the relay server /// protocol. pub(crate) fn add_potential_candidates( @@ -95,7 +117,7 @@ impl RelayManager { /// Try connecting to candidate relays if we are below the threshold connections. /// This is run periodically on a loop. pub(crate) fn try_connecting_to_relay(&mut self, swarm: &mut Swarm) { - if !self.we_are_behind_nat { + if !self.enabled { return; } diff --git a/sn_node/src/bin/safenode/main.rs b/sn_node/src/bin/safenode/main.rs index 7659f51a93..482670ed5b 100644 --- a/sn_node/src/bin/safenode/main.rs +++ b/sn_node/src/bin/safenode/main.rs @@ -67,6 +67,13 @@ pub fn parse_log_output(val: &str) -> Result { #[derive(Parser, Debug)] #[clap(name = "safenode cli", version = env!("CARGO_PKG_VERSION"))] struct Opt { + /// Specify whether the node is operating from a home network and situated behind a NAT without port forwarding + /// capabilities. Setting this to true, activates hole-punching to facilitate direct connections from other nodes. + /// + /// If this not enabled and you're behind a NAT, the node is terminated. + #[clap(long, default_value_t = false)] + home_network: bool, + /// Specify the logging output destination. /// /// Valid values are "stdout", "data-dir", or a custom path. @@ -179,13 +186,14 @@ fn main() -> Result<()> { #[cfg(feature = "metrics")] rt.spawn(init_metrics(std::process::id())); let restart_options = rt.block_on(async move { - let node_builder = NodeBuilder::new( + let mut node_builder = NodeBuilder::new( keypair, node_socket_addr, bootstrap_peers, opt.local, root_dir, ); + node_builder.enable_hole_punching = opt.home_network; #[cfg(feature = "open-metrics")] let mut node_builder = node_builder; #[cfg(feature = "open-metrics")] @@ -336,11 +344,11 @@ fn monitor_node_events(mut node_events_rx: NodeEventsReceiver, ctrl_tx: mpsc::Se break; } } - Ok(NodeEvent::TerminateNode) => { + Ok(NodeEvent::TerminateNode(reason)) => { if let Err(err) = ctrl_tx .send(NodeCtrl::Stop { delay: Duration::from_secs(1), - cause: eyre!("Node terminated due to termination command !"), + cause: eyre!("Node terminated due to: {reason:?}"), }) .await { diff --git a/sn_node/src/event.rs b/sn_node/src/event.rs index 68a53a7e4c..0f74995770 100644 --- a/sn_node/src/event.rs +++ b/sn_node/src/event.rs @@ -67,7 +67,7 @@ pub enum NodeEvent { /// One of the sub event channel closed and unrecoverable. ChannelClosed, /// Terminates the node - TerminateNode, + TerminateNode(String), } impl NodeEvent { diff --git a/sn_node/src/node.rs b/sn_node/src/node.rs index c19f6f9b13..8fa5a76674 100644 --- a/sn_node/src/node.rs +++ b/sn_node/src/node.rs @@ -65,6 +65,8 @@ pub struct NodeBuilder { root_dir: PathBuf, #[cfg(feature = "open-metrics")] metrics_server_port: u16, + /// Enable hole punching for nodes connecting from home networks. + pub enable_hole_punching: bool, } impl NodeBuilder { @@ -84,6 +86,7 @@ impl NodeBuilder { root_dir, #[cfg(feature = "open-metrics")] metrics_server_port: 0, + enable_hole_punching: false, } } @@ -135,6 +138,7 @@ impl NodeBuilder { #[cfg(feature = "open-metrics")] network_builder.metrics_server_port(self.metrics_server_port); network_builder.initial_peers(self.initial_peers.clone()); + network_builder.enable_hole_punching(self.enable_hole_punching); let (network, network_event_receiver, swarm_driver) = network_builder.build_node()?; let node_events_channel = NodeEventsChannel::default(); @@ -404,10 +408,11 @@ impl Node { }); } - NetworkEvent::TerminateNode => { + NetworkEvent::TerminateNode { reason } => { event_header = "TerminateNode"; - error!("Received termination from swarm_driver due to too many HDD write errors."); - self.events_channel.broadcast(NodeEvent::TerminateNode); + error!("Received termination from swarm_driver due to {reason:?}"); + self.events_channel + .broadcast(NodeEvent::TerminateNode(format!("{reason:?}"))); } NetworkEvent::FailedToFetchHolders(bad_nodes) => { event_header = "FailedToFetchHolders"; From 98ddc7fdb63b3a6c65784b4476d9b405611ced7f Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 10 Apr 2024 19:15:00 +0530 Subject: [PATCH 107/205] chore: use quic again --- sn_networking/src/driver.rs | 18 ++++++------------ sn_peers_acquisition/src/lib.rs | 4 +++- sn_protocol/src/lib.rs | 2 +- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index 1ec7339078..297e062ae4 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -313,11 +313,12 @@ impl NetworkBuilder { // Listen on the provided address let listen_socket_addr = listen_addr.ok_or(NetworkError::ListenAddressNotProvided)?; - // Listen on TCP for autonat - let addr_tcp = - Multiaddr::from(listen_socket_addr.ip()).with(Protocol::Tcp(listen_socket_addr.port())); + // Listen on QUIC + let addr_quic = Multiaddr::from(listen_socket_addr.ip()) + .with(Protocol::Udp(listen_socket_addr.port())) + .with(Protocol::QuicV1); swarm_driver - .listen_on(addr_tcp) + .listen_on(addr_quic) .expect("Multiaddr should be supported by our configured transports"); // Listen on WebSocket @@ -462,14 +463,7 @@ impl NetworkBuilder { let swarm = libp2p::SwarmBuilder::with_existing_identity(self.keypair.clone()) .with_tokio() - .with_tcp( - libp2p::tcp::Config::default() - .port_reuse(true) - .nodelay(true), - libp2p::noise::Config::new, - libp2p::yamux::Config::default, - ) - .map_err(|e| NetworkError::BahviourErr(e.to_string()))? + .with_quic_config(|_config| libp2p::quic::Config::new(&self.keypair)) .with_relay_client(libp2p::noise::Config::new, libp2p::yamux::Config::default) .map_err(|e| NetworkError::BahviourErr(e.to_string()))? .with_behaviour(|_keypair, relay_behaviour| NodeBehaviour { diff --git a/sn_peers_acquisition/src/lib.rs b/sn_peers_acquisition/src/lib.rs index a9d7035fa2..e1698ea374 100644 --- a/sn_peers_acquisition/src/lib.rs +++ b/sn_peers_acquisition/src/lib.rs @@ -137,7 +137,9 @@ pub fn parse_peer_addr(addr: &str) -> Result { if let Ok(addr) = addr.parse::() { let start_addr = Multiaddr::from(*addr.ip()); // Start with an address into a `/ip4//udp//quic-v1` multiaddr. - let multiaddr = start_addr.with(Protocol::Tcp(addr.port())); + let multiaddr = start_addr + .with(Protocol::Udp(addr.port())) + .with(Protocol::QuicV1); #[cfg(all(feature = "websockets", feature = "wasm32"))] // Turn the address into a `/ip4//udp//websocket-websys-v1` multiaddr. diff --git a/sn_protocol/src/lib.rs b/sn_protocol/src/lib.rs index 7b93a05421..fc3657c825 100644 --- a/sn_protocol/src/lib.rs +++ b/sn_protocol/src/lib.rs @@ -48,7 +48,7 @@ use xor_name::XorName; pub fn get_port_from_multiaddr(multi_addr: &Multiaddr) -> Option { // assuming the listening addr contains /ip4/127.0.0.1/udp/56215/quic-v1/p2p/ for protocol in multi_addr.iter() { - if let Protocol::Tcp(port) = protocol { + if let Protocol::Udp(port) = protocol { return Some(port); } } From 7a472899da6c458f69d2312a1ce9460033d94985 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Thu, 11 Apr 2024 14:52:40 +0530 Subject: [PATCH 108/205] fix(relay): craft the correctly formatted relay address --- sn_networking/src/event.rs | 8 ++++-- sn_networking/src/relay_manager.rs | 46 ++++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 573138dc8a..118132b7cf 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -412,9 +412,11 @@ impl SwarmDriver { } if self.relay_manager.are_we_behind_nat(&mut self.swarm) { - self.send_event(NetworkEvent::TerminateNode { - reason: TerminateNodeReason::BehindNAT, - }) + error!("Node reported as being behind NAT for not having enough incoming connections. This can be false positive."); + + // self.send_event(NetworkEvent::TerminateNode { + // reason: TerminateNodeReason::BehindNAT, + // }) } } } diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index ec4f5a11fc..f878884b79 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -39,7 +39,9 @@ impl RelayManager { .filter_map(|addr| { for protocol in addr.iter() { if let Protocol::P2p(peer_id) = protocol { - return Some((peer_id, addr)); + let relay_addr = Self::craft_relay_address(&addr, Some(peer_id))?; + + return Some((peer_id, relay_addr)); } } None @@ -106,8 +108,12 @@ impl RelayManager { if let Some(addr) = addrs.iter().next() { // only consider non relayed peers if !addr.iter().any(|p| p == Protocol::P2pCircuit) { - debug!("Adding {peer_id:?} with {addr:?} as a potential relay candidate"); - self.candidates.push_back((*peer_id, addr.clone())); + if let Some(relay_addr) = Self::craft_relay_address(addr, None) { + debug!( + "Adding {peer_id:?} with {relay_addr:?} as a potential relay candidate" + ); + self.candidates.push_back((*peer_id, relay_addr)); + } } } } @@ -133,8 +139,7 @@ impl RelayManager { while n_reservations < reservations_to_make { // todo: should we remove all our other `listen_addr`? And should we block from adding `add_external_address` if // we're behind nat? - if let Some((peer_id, addr)) = self.candidates.pop_front() { - let relay_addr = addr.with(Protocol::P2pCircuit); + if let Some((peer_id, relay_addr)) = self.candidates.pop_front() { match swarm.listen_on(relay_addr.clone()) { Ok(id) => { info!("Sending reservation to relay {peer_id:?} on {relay_addr:?}"); @@ -201,4 +206,35 @@ impl RelayManager { } false } + + // the listen addr should be something like `/ip4/198.51.100.0/tcp/55555/p2p/QmRelay/p2p-circuit/ + fn craft_relay_address(addr: &Multiaddr, peer_id: Option) -> Option { + let mut output_addr = Multiaddr::empty(); + + let ip = addr + .iter() + .find(|protocol| matches!(protocol, Protocol::Ip4(_)))?; + output_addr.push(ip); + let port = addr + .iter() + .find(|protocol| matches!(protocol, Protocol::Udp(_)))?; + output_addr.push(port); + output_addr.push(Protocol::QuicV1); + + let peer_id = { + if let Some(peer_id) = peer_id { + Protocol::P2p(peer_id) + } else { + let peer_id = addr + .iter() + .find(|protocol| matches!(protocol, Protocol::P2p(_)))?; + peer_id + } + }; + output_addr.push(peer_id); + output_addr.push(Protocol::P2pCircuit); + + debug!("Crafted p2p relay address: {output_addr:?}"); + Some(output_addr) + } } From b0ee5b5103e1c42b8890ecb82566805b2bed0f88 Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Thu, 11 Apr 2024 11:22:03 +0200 Subject: [PATCH 109/205] refactor(networking): re-add global_only --- sn_networking/src/ListenBehaviour.rs | 0 sn_networking/src/driver.rs | 68 ++++++++++++++++++++-------- 2 files changed, 50 insertions(+), 18 deletions(-) create mode 100644 sn_networking/src/ListenBehaviour.rs diff --git a/sn_networking/src/ListenBehaviour.rs b/sn_networking/src/ListenBehaviour.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index 297e062ae4..3d3b078404 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -10,7 +10,6 @@ use crate::metrics::NetworkMetrics; #[cfg(feature = "open-metrics")] use crate::metrics_service::run_metrics_server; -use crate::NodeIssue; use crate::{ bootstrap::{ContinuousBootstrap, BOOTSTRAP_INTERVAL}, circular_vec::CircularVec, @@ -27,9 +26,13 @@ use crate::{ target_arch::{interval, spawn, Instant}, Network, CLOSE_GROUP_SIZE, }; +use crate::{transport, NodeIssue}; +use futures::future::Either; use futures::StreamExt; +use libp2p::core::muxing::StreamMuxerBox; #[cfg(feature = "local-discovery")] use libp2p::mdns; +use libp2p::Transport as _; use libp2p::{ identity::Keypair, kad::{self, QueryId, Quorum, Record, K_VALUE}, @@ -461,24 +464,53 @@ impl NetworkBuilder { libp2p::identify::Behaviour::new(cfg) }; - let swarm = libp2p::SwarmBuilder::with_existing_identity(self.keypair.clone()) - .with_tokio() - .with_quic_config(|_config| libp2p::quic::Config::new(&self.keypair)) - .with_relay_client(libp2p::noise::Config::new, libp2p::yamux::Config::default) - .map_err(|e| NetworkError::BahviourErr(e.to_string()))? - .with_behaviour(|_keypair, relay_behaviour| NodeBehaviour { - relay_client: relay_behaviour, - relay_server: libp2p::relay::Behaviour::new(peer_id, Default::default()), - request_response, - #[cfg(feature = "local-discovery")] - mdns, - identify, - kademlia, - dcutr: libp2p::dcutr::Behaviour::new(peer_id), + let main_transport = transport::build_transport(&self.keypair); + + let transport = if !self.local { + debug!("Preventing non-global dials"); + // Wrap upper in a transport that prevents dialing local addresses. + libp2p::core::transport::global_only::Transport::new(main_transport).boxed() + } else { + main_transport + }; + + let (relay_transport, relay_behaviour) = + libp2p::relay::client::new(self.keypair.public().to_peer_id()); + let relay_transport = relay_transport + .upgrade(libp2p::core::upgrade::Version::V1Lazy) + .authenticate( + libp2p::noise::Config::new(&self.keypair) + .expect("Signing libp2p-noise static DH keypair failed."), + ) + .multiplex(libp2p::yamux::Config::default()) + .or_transport(transport); + + let transport = relay_transport + .map(|either_output, _| match either_output { + Either::Left((peer_id, muxer)) => (peer_id, StreamMuxerBox::new(muxer)), + Either::Right((peer_id, muxer)) => (peer_id, StreamMuxerBox::new(muxer)), }) - .map_err(|e| NetworkError::BahviourErr(e.to_string()))? - .with_swarm_config(|c| c.with_idle_connection_timeout(CONNECTION_KEEP_ALIVE_TIMEOUT)) - .build(); + .boxed(); + + let behaviour = NodeBehaviour { + relay_client: relay_behaviour, + relay_server: libp2p::relay::Behaviour::new(peer_id, Default::default()), + request_response, + kademlia, + identify, + #[cfg(feature = "local-discovery")] + mdns, + dcutr: libp2p::dcutr::Behaviour::new(peer_id), + }; + + #[cfg(not(target_arch = "wasm32"))] + let swarm_config = libp2p::swarm::Config::with_tokio_executor() + .with_idle_connection_timeout(CONNECTION_KEEP_ALIVE_TIMEOUT); + #[cfg(target_arch = "wasm32")] + let swarm_config = libp2p::swarm::Config::with_wasm_executor() + .with_idle_connection_timeout(CONNECTION_KEEP_ALIVE_TIMEOUT); + + let swarm = Swarm::new(transport, behaviour, peer_id, swarm_config); let bootstrap = ContinuousBootstrap::new(); let replication_fetcher = ReplicationFetcher::new(peer_id, network_event_sender.clone()); From 852a87df68663e6fe2ea022064d2d7299e3ba604 Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Thu, 11 Apr 2024 11:30:55 +0200 Subject: [PATCH 110/205] chore(networking): remove empty file --- sn_networking/src/ListenBehaviour.rs | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 sn_networking/src/ListenBehaviour.rs diff --git a/sn_networking/src/ListenBehaviour.rs b/sn_networking/src/ListenBehaviour.rs deleted file mode 100644 index e69de29bb2..0000000000 From 41e868a31872f264d9e7faa021db8ab55984d6b4 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Thu, 11 Apr 2024 15:04:04 +0530 Subject: [PATCH 111/205] chore: clippy fix --- sn_networking/src/relay_manager.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index f878884b79..42f158b7f2 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -225,10 +225,8 @@ impl RelayManager { if let Some(peer_id) = peer_id { Protocol::P2p(peer_id) } else { - let peer_id = addr - .iter() - .find(|protocol| matches!(protocol, Protocol::P2p(_)))?; - peer_id + addr.iter() + .find(|protocol| matches!(protocol, Protocol::P2p(_)))? } }; output_addr.push(peer_id); From 640665d0aacdfb1250246ecbf70be1f1b4222277 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Thu, 11 Apr 2024 16:48:16 +0530 Subject: [PATCH 112/205] fix(network): do not strip out relay's PeerId --- sn_networking/src/lib.rs | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/sn_networking/src/lib.rs b/sn_networking/src/lib.rs index 314750af54..8042a5e0ca 100644 --- a/sn_networking/src/lib.rs +++ b/sn_networking/src/lib.rs @@ -934,11 +934,30 @@ pub(crate) fn multiaddr_pop_p2p(multiaddr: &mut Multiaddr) -> Option { } /// Build a `Multiaddr` with the p2p protocol filtered out. +/// If it is a relayed address, then the relay's P2P address is preserved. pub(crate) fn multiaddr_strip_p2p(multiaddr: &Multiaddr) -> Multiaddr { - multiaddr - .iter() - .filter(|p| !matches!(p, Protocol::P2p(_))) - .collect() + let is_relayed = multiaddr.iter().any(|p| matches!(p, Protocol::P2pCircuit)); + + if is_relayed { + // Do not add any PeerId after we've found the P2PCircuit protocol. The prior one is the relay's PeerId which + // we should preserve. + let mut before_relay_protocol = true; + let mut new_multi_addr = Multiaddr::empty(); + for p in multiaddr.iter() { + if matches!(p, Protocol::P2pCircuit) { + before_relay_protocol = false; + } + if matches!(p, Protocol::P2p(_)) && before_relay_protocol { + new_multi_addr.push(p); + } + } + new_multi_addr + } else { + multiaddr + .iter() + .filter(|p| !matches!(p, Protocol::P2p(_))) + .collect() + } } pub(crate) fn send_swarm_cmd(swarm_cmd_sender: Sender, cmd: SwarmCmd) { From 317bf1e36192ab4a6aa94ff951e64deba084fb48 Mon Sep 17 00:00:00 2001 From: Benno Zeeman Date: Thu, 11 Apr 2024 14:16:35 +0200 Subject: [PATCH 113/205] fix(networking): do not add to dialed peers --- sn_networking/src/event.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 118132b7cf..797c50c74d 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -371,7 +371,6 @@ impl SwarmDriver { if !kbucket_full { info!(%peer_id, ?addrs, "received identify info from undialed peer for not full kbucket {ilog2:?}, dial back to confirm external accessible"); - self.dialed_peers.push(peer_id); if let Err(err) = self.swarm.dial( DialOpts::peer_id(peer_id) .condition(PeerCondition::NotDialing) From 53296c8b451a4e2d620092f3be9f0b2639e192b8 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Fri, 12 Apr 2024 03:25:37 +0530 Subject: [PATCH 114/205] fix: do not add reported external addressese if we are behind home network --- sn_networking/src/driver.rs | 12 +++++++----- sn_networking/src/event.rs | 19 +++++++++++++++---- sn_networking/src/relay_manager.rs | 5 +++-- sn_node/src/bin/safenode/main.rs | 2 +- sn_node/src/node.rs | 6 +++--- 5 files changed, 29 insertions(+), 15 deletions(-) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index 3d3b078404..a39cba8415 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -183,7 +183,7 @@ pub(super) struct NodeBehaviour { #[derive(Debug)] pub struct NetworkBuilder { - enable_hole_punching: bool, + is_behind_home_network: bool, keypair: Keypair, local: bool, root_dir: PathBuf, @@ -200,7 +200,7 @@ pub struct NetworkBuilder { impl NetworkBuilder { pub fn new(keypair: Keypair, local: bool, root_dir: PathBuf) -> Self { Self { - enable_hole_punching: false, + is_behind_home_network: false, keypair, local, root_dir, @@ -215,8 +215,8 @@ impl NetworkBuilder { } } - pub fn enable_hole_punching(&mut self, enable: bool) { - self.enable_hole_punching = enable; + pub fn is_behind_home_network(&mut self, enable: bool) { + self.is_behind_home_network = enable; } pub fn listen_addr(&mut self, listen_addr: SocketAddr) { @@ -516,7 +516,7 @@ impl NetworkBuilder { let replication_fetcher = ReplicationFetcher::new(peer_id, network_event_sender.clone()); let mut relay_manager = RelayManager::new(self.initial_peers); if !is_client { - relay_manager.enable_hole_punching(self.enable_hole_punching); + relay_manager.enable_hole_punching(self.is_behind_home_network); } let swarm_driver = SwarmDriver { @@ -525,6 +525,7 @@ impl NetworkBuilder { local: self.local, listen_port: self.listen_addr.map(|addr| addr.port()), is_client, + is_behind_home_network: self.is_behind_home_network, connected_peers: 0, bootstrap, relay_manager, @@ -569,6 +570,7 @@ pub struct SwarmDriver { pub(crate) self_peer_id: PeerId, pub(crate) local: bool, pub(crate) is_client: bool, + pub(crate) is_behind_home_network: bool, /// The port that was set by the user pub(crate) listen_port: Option, pub(crate) connected_peers: usize, diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 797c50c74d..6366497bf3 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -327,7 +327,7 @@ impl SwarmDriver { .collect(), }; - if !self.local && peer_is_node { + if peer_is_node { // add potential relay candidates. self.relay_manager.add_potential_candidates( &peer_id, @@ -470,8 +470,9 @@ impl SwarmDriver { let local_peer_id = *self.swarm.local_peer_id(); let address = address.with(Protocol::P2p(local_peer_id)); - // Trigger server mode if we're not a client - if !self.is_client { + // Trigger server mode if we're not a client and we should not add our own address if we're behind + // home network. + if !self.is_client && !self.is_behind_home_network { if self.local { // all addresses are effectively external here... // this is needed for Kad Mode::Server @@ -684,7 +685,17 @@ impl SwarmDriver { SwarmEvent::NewExternalAddrCandidate { address } => { event_string = "NewExternalAddrCandidate"; - if !self.swarm.external_addresses().any(|addr| addr == &address) && !self.is_client + let all_external_addresses = self.swarm.external_addresses().collect_vec(); + let all_listeners = self.swarm.listeners().collect_vec(); + info!("All our listeners: {all_listeners:?}"); + info!("All our external addresses: {all_external_addresses:?}"); + + if !self.swarm.external_addresses().any(|addr| addr == &address) + && !self.is_client + // If we are behind a home network, then our IP is returned here. We should be only having + // relay server as our external address + // todo: can our relay address be reported here? If so, maybe we should add them. + && !self.is_behind_home_network { debug!(%address, "external address: new candidate"); diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index 42f158b7f2..2951ffab44 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -168,13 +168,14 @@ impl RelayManager { while !self.non_relayed_listener_id.is_empty() { if let Some(listener_id) = self.non_relayed_listener_id.pop_back() { let res = swarm.remove_listener(listener_id); - debug!("Removed {listener_id:?} with result: {res} from swarm as we now have a relay reservation"); + debug!("Successful reservation: Removing {listener_id:?} with result: {res} from swarm as we now have a relay reservation"); } } match self.waiting_for_reservation.remove(peer_id) { Some(addr) => { - info!("Successfully made reservation with {peer_id:?} on {addr:?}"); + info!("Successfully made reservation with {peer_id:?} on {addr:?}. Adding the addr to external address."); + swarm.add_external_address(addr.clone()); self.connected_relays.insert(*peer_id, addr); } None => { diff --git a/sn_node/src/bin/safenode/main.rs b/sn_node/src/bin/safenode/main.rs index 482670ed5b..56b337ce3c 100644 --- a/sn_node/src/bin/safenode/main.rs +++ b/sn_node/src/bin/safenode/main.rs @@ -193,7 +193,7 @@ fn main() -> Result<()> { opt.local, root_dir, ); - node_builder.enable_hole_punching = opt.home_network; + node_builder.is_behind_home_network = opt.home_network; #[cfg(feature = "open-metrics")] let mut node_builder = node_builder; #[cfg(feature = "open-metrics")] diff --git a/sn_node/src/node.rs b/sn_node/src/node.rs index 8fa5a76674..cc3e6a920f 100644 --- a/sn_node/src/node.rs +++ b/sn_node/src/node.rs @@ -66,7 +66,7 @@ pub struct NodeBuilder { #[cfg(feature = "open-metrics")] metrics_server_port: u16, /// Enable hole punching for nodes connecting from home networks. - pub enable_hole_punching: bool, + pub is_behind_home_network: bool, } impl NodeBuilder { @@ -86,7 +86,7 @@ impl NodeBuilder { root_dir, #[cfg(feature = "open-metrics")] metrics_server_port: 0, - enable_hole_punching: false, + is_behind_home_network: false, } } @@ -138,7 +138,7 @@ impl NodeBuilder { #[cfg(feature = "open-metrics")] network_builder.metrics_server_port(self.metrics_server_port); network_builder.initial_peers(self.initial_peers.clone()); - network_builder.enable_hole_punching(self.enable_hole_punching); + network_builder.is_behind_home_network(self.is_behind_home_network); let (network, network_event_receiver, swarm_driver) = network_builder.build_node()?; let node_events_channel = NodeEventsChannel::default(); From 4a1fac4cc72faae0453102ff9b951f0ffe818ada Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Fri, 12 Apr 2024 04:19:10 +0530 Subject: [PATCH 115/205] fix(relay): crafted multi address should contain the P2PCircuit protocol --- sn_networking/src/lib.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sn_networking/src/lib.rs b/sn_networking/src/lib.rs index 8042a5e0ca..afbf50930d 100644 --- a/sn_networking/src/lib.rs +++ b/sn_networking/src/lib.rs @@ -947,9 +947,10 @@ pub(crate) fn multiaddr_strip_p2p(multiaddr: &Multiaddr) -> Multiaddr { if matches!(p, Protocol::P2pCircuit) { before_relay_protocol = false; } - if matches!(p, Protocol::P2p(_)) && before_relay_protocol { - new_multi_addr.push(p); + if matches!(p, Protocol::P2p(_)) && !before_relay_protocol { + continue; } + new_multi_addr.push(p); } new_multi_addr } else { From 81ef249181fa02a81036f59400e4896723389e80 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Fri, 12 Apr 2024 05:36:19 +0530 Subject: [PATCH 116/205] chore: do not remove old non-relayed listeners --- sn_networking/src/relay_manager.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index 2951ffab44..bc4af9c901 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -165,12 +165,12 @@ impl RelayManager { swarm: &mut Swarm, ) { // now that we have made a reservation, remove our non-relayed listeners - while !self.non_relayed_listener_id.is_empty() { - if let Some(listener_id) = self.non_relayed_listener_id.pop_back() { - let res = swarm.remove_listener(listener_id); - debug!("Successful reservation: Removing {listener_id:?} with result: {res} from swarm as we now have a relay reservation"); - } - } + // while !self.non_relayed_listener_id.is_empty() { + // if let Some(listener_id) = self.non_relayed_listener_id.pop_back() { + // let res = swarm.remove_listener(listener_id); + // debug!("Successful reservation: Removing {listener_id:?} with result: {res} from swarm as we now have a relay reservation"); + // } + // } match self.waiting_for_reservation.remove(peer_id) { Some(addr) => { From 4cf13fefb00d917ef8193b8a7ca23d69a0379f54 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Fri, 12 Apr 2024 05:40:17 +0530 Subject: [PATCH 117/205] chore: add debug lines while adding potential relay candidates --- sn_networking/src/relay_manager.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index bc4af9c901..3d5d67f292 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -99,7 +99,9 @@ impl RelayManager { addrs: &HashSet, stream_protocols: &Vec, ) { + trace!("Trying to add potential relay candidates for {peer_id:?} with addrs: {addrs:?}"); if self.candidates.len() >= MAX_POTENTIAL_CANDIDATES { + trace!("Got max relay candidates"); return; } @@ -113,9 +115,15 @@ impl RelayManager { "Adding {peer_id:?} with {relay_addr:?} as a potential relay candidate" ); self.candidates.push_back((*peer_id, relay_addr)); + } else { + trace!("Was not able to craft relay address"); } + } else { + trace!("Addr contains P2pCircuit protocol. Not adding as candidate."); } } + } else { + trace!("Peer does not support relay server protocol"); } } From 6f3b4286142e59fe9e287ef5b7ccdc2ea67ccf27 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Fri, 12 Apr 2024 16:40:30 +0530 Subject: [PATCH 118/205] fix: pass peer id while crafting relay address --- sn_networking/src/event.rs | 2 +- sn_networking/src/relay_manager.rs | 11 ++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 6366497bf3..2d30c80746 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -487,7 +487,7 @@ impl SwarmDriver { self.send_event(NetworkEvent::NewListenAddr(address.clone())); - info!("Local node is listening (id: {listener_id}) on {address:?}"); + info!("Local node is listening {listener_id:?} on {address:?}"); } SwarmEvent::ListenerClosed { listener_id, diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index 3d5d67f292..255512d377 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -110,7 +110,7 @@ impl RelayManager { if let Some(addr) = addrs.iter().next() { // only consider non relayed peers if !addr.iter().any(|p| p == Protocol::P2pCircuit) { - if let Some(relay_addr) = Self::craft_relay_address(addr, None) { + if let Some(relay_addr) = Self::craft_relay_address(addr, Some(*peer_id)) { debug!( "Adding {peer_id:?} with {relay_addr:?} as a potential relay candidate" ); @@ -172,14 +172,6 @@ impl RelayManager { peer_id: &PeerId, swarm: &mut Swarm, ) { - // now that we have made a reservation, remove our non-relayed listeners - // while !self.non_relayed_listener_id.is_empty() { - // if let Some(listener_id) = self.non_relayed_listener_id.pop_back() { - // let res = swarm.remove_listener(listener_id); - // debug!("Successful reservation: Removing {listener_id:?} with result: {res} from swarm as we now have a relay reservation"); - // } - // } - match self.waiting_for_reservation.remove(peer_id) { Some(addr) => { info!("Successfully made reservation with {peer_id:?} on {addr:?}. Adding the addr to external address."); @@ -193,6 +185,7 @@ impl RelayManager { } /// Update our state if the reservation has been cancelled or if the relay has closed. + /// todo: remove from external addresses. pub(crate) fn update_on_listener_closed(&mut self, listener_id: &ListenerId) { let Some(peer_id) = self.relayed_listener_id_map.remove(listener_id) else { return; From eac75b07c38d2e0b6396cb10c42b00deaf1aa5b6 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Mon, 15 Apr 2024 10:40:21 +0530 Subject: [PATCH 119/205] fix: keep idle connections forever --- sn_networking/src/driver.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index a39cba8415..0982709397 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -93,7 +93,7 @@ const MAX_PACKET_SIZE: usize = 1024 * 1024 * 5; // the chunk size is 1mb, so sho // Timeout for requests sent/received through the request_response behaviour. const REQUEST_TIMEOUT_DEFAULT_S: Duration = Duration::from_secs(30); // Sets the keep-alive timeout of idle connections. -const CONNECTION_KEEP_ALIVE_TIMEOUT: Duration = Duration::from_secs(30); +const CONNECTION_KEEP_ALIVE_TIMEOUT: Duration = Duration::from_secs(u64::MAX); const NETWORKING_CHANNEL_SIZE: usize = 10_000; From cf4561e52ad05244684fad6c996436979623e797 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Mon, 15 Apr 2024 10:40:37 +0530 Subject: [PATCH 120/205] chore(tryout): do not add new relay candidates --- sn_networking/src/event.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 2d30c80746..3e76fd4065 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -327,14 +327,14 @@ impl SwarmDriver { .collect(), }; - if peer_is_node { - // add potential relay candidates. - self.relay_manager.add_potential_candidates( - &peer_id, - &addrs, - &info.protocols, - ); - } + // if peer_is_node { + // // add potential relay candidates. + // self.relay_manager.add_potential_candidates( + // &peer_id, + // &addrs, + // &info.protocols, + // ); + // } // When received an identify from un-dialed peer, try to dial it // The dial shall trigger the same identify to be sent again and confirm From b36855d0fce06460e81b3522152aa9465f9120b5 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Mon, 15 Apr 2024 13:57:27 +0530 Subject: [PATCH 121/205] fix: increase relay server capacity --- sn_networking/src/driver.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index 0982709397..8401f44c93 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -29,10 +29,10 @@ use crate::{ use crate::{transport, NodeIssue}; use futures::future::Either; use futures::StreamExt; -use libp2p::core::muxing::StreamMuxerBox; #[cfg(feature = "local-discovery")] use libp2p::mdns; use libp2p::Transport as _; +use libp2p::{core::muxing::StreamMuxerBox, relay}; use libp2p::{ identity::Keypair, kad::{self, QueryId, Quorum, Record, K_VALUE}, @@ -492,9 +492,19 @@ impl NetworkBuilder { }) .boxed(); + let relay_server = { + let relay_server_cfg = relay::Config { + max_reservations: 1024, // the number of home nodes that we can support + max_circuits: 32_000, // total max number of relayed connections we can support + max_circuits_per_peer: 1024, // max number of relayed connections per peer + ..Default::default() + }; + libp2p::relay::Behaviour::new(peer_id, relay_server_cfg) + }; + let behaviour = NodeBehaviour { relay_client: relay_behaviour, - relay_server: libp2p::relay::Behaviour::new(peer_id, Default::default()), + relay_server, request_response, kademlia, identify, From 95271592efcb37168b504cfccd43b4ca4b48b64e Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Mon, 22 Apr 2024 23:17:08 +0530 Subject: [PATCH 122/205] fix: do not remove outdated connections --- sn_networking/src/event.rs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 3e76fd4065..cfd6358200 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -309,6 +309,11 @@ impl SwarmDriver { let has_dialed = self.dialed_peers.contains(&peer_id); let peer_is_node = info.agent_version == IDENTIFY_NODE_VERSION_STR.to_string(); + trace!( + ?has_dialed, + ?peer_is_node, + "identify: state for {peer_id:?}" + ); // If we're not in local mode, only add globally reachable addresses. // Strip the `/p2p/...` part of the multiaddresses. @@ -379,6 +384,8 @@ impl SwarmDriver { ) { warn!(%peer_id, ?addrs, "dialing error: {err:?}"); } + } else { + trace!("received identify for a full bucket {ilog2:?}, no dialing {peer_id:?} on {addrs:?}"); } trace!( @@ -539,6 +546,14 @@ impl SwarmDriver { trace!(%peer_id, ?connection_id, ?cause, num_established, "ConnectionClosed: {}", endpoint_str(&endpoint)); let _ = self.live_connected_peers.remove(&connection_id); } + SwarmEvent::OutgoingConnectionError { + connection_id, + peer_id: None, + error, + } => { + event_string = "OutgoingConnErr"; + warn!("OutgoingConnectionError to on {connection_id:?} - {error:?}"); + } SwarmEvent::OutgoingConnectionError { peer_id: Some(failed_peer_id), error, @@ -733,8 +748,6 @@ impl SwarmDriver { } } - self.remove_outdated_connections(); - self.log_handling(event_string.to_string(), start.elapsed()); trace!( @@ -1266,7 +1279,7 @@ impl SwarmDriver { } // Remove outdated connection to a peer if it is not in the RT. - fn remove_outdated_connections(&mut self) { + fn _remove_outdated_connections(&mut self) { let mut shall_removed = vec![]; self.live_connected_peers From dad1fdd8161f0595889ccde712c5c47244fbda5a Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 23 Apr 2024 00:14:47 +0530 Subject: [PATCH 123/205] chore: return early if peer is not a node --- sn_networking/src/driver.rs | 2 +- sn_networking/src/event.rs | 28 +++++++--------------------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index 8401f44c93..3bb5a08d10 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -601,7 +601,7 @@ pub struct SwarmDriver { pub(crate) pending_requests: HashMap>>>, pub(crate) pending_get_record: PendingGetRecord, - /// A list of the most recent peers we have dialed ourselves. + /// A list of the most recent peers we have dialed ourselves. Old dialed peers are evicted once the vec fills up. pub(crate) dialed_peers: CircularVec, // A list of random `PeerId` candidates that falls into kbuckets, // This is to ensure a more accurate network discovery. diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index cfd6358200..0496fddb13 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -279,19 +279,7 @@ impl SwarmDriver { SwarmEvent::Behaviour(NodeEvent::Identify(iden)) => { event_string = "identify"; - // TODO: in identify, we first need to know if we are behind nat. - // 1: TCP connections only, we do run autonat request in the background. - // - // (Which means all nodes must listen and act on autonat for tcp conns only.) - // - // 2. If we are behind nat and unreachable, we should not dial other nodes for now. - // 3. If we are unreachable, we connect to a relay server. - // 4. We then use dcutr to connect to other nodes and register valid quic connections. - // 5. That is holes punched? - - // Match on the Identify event. match *iden { - // If the event is a Received event, handle the received peer information. libp2p::identify::Event::Received { peer_id, info } => { trace!(%peer_id, ?info, "identify: received info"); @@ -307,13 +295,11 @@ impl SwarmDriver { } let has_dialed = self.dialed_peers.contains(&peer_id); - let peer_is_node = - info.agent_version == IDENTIFY_NODE_VERSION_STR.to_string(); - trace!( - ?has_dialed, - ?peer_is_node, - "identify: state for {peer_id:?}" - ); + + // if client, return. + if info.agent_version != IDENTIFY_NODE_VERSION_STR.to_string() { + return Ok(()); + } // If we're not in local mode, only add globally reachable addresses. // Strip the `/p2p/...` part of the multiaddresses. @@ -344,7 +330,7 @@ impl SwarmDriver { // When received an identify from un-dialed peer, try to dial it // The dial shall trigger the same identify to be sent again and confirm // peer is external accessible, hence safe to be added into RT. - if !self.local && peer_is_node && !has_dialed { + if !self.local && !has_dialed { // Only need to dial back for not fulfilled kbucket let (kbucket_full, ilog2) = if let Some(kbucket) = self.swarm.behaviour_mut().kademlia.kbucket(peer_id) @@ -396,7 +382,7 @@ impl SwarmDriver { } // If we are not local, we care only for peers that we dialed and thus are reachable. - if self.local || has_dialed && peer_is_node { + if self.local || has_dialed { // To reduce the bad_node check resource usage, // during the connection establish process, only check cached black_list // The periodical check, which involves network queries shall filter From 68e91bd969c574beff11ff60ff8a6f9c778d5f7f Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 23 Apr 2024 00:26:38 +0530 Subject: [PATCH 124/205] chore: enable multiple relay connections --- sn_networking/src/event.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 0496fddb13..95ce69008c 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -318,14 +318,11 @@ impl SwarmDriver { .collect(), }; - // if peer_is_node { - // // add potential relay candidates. - // self.relay_manager.add_potential_candidates( - // &peer_id, - // &addrs, - // &info.protocols, - // ); - // } + self.relay_manager.add_potential_candidates( + &peer_id, + &addrs, + &info.protocols, + ); // When received an identify from un-dialed peer, try to dial it // The dial shall trigger the same identify to be sent again and confirm From 14a6a4876835906f67e2df1d4778424e8dff3247 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 23 Apr 2024 18:07:52 +0530 Subject: [PATCH 125/205] fix: update outdated connection removal flow --- sn_networking/src/event.rs | 77 +++++++++++++++++------------- sn_networking/src/relay_manager.rs | 4 ++ 2 files changed, 48 insertions(+), 33 deletions(-) diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 95ce69008c..7223318394 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -494,7 +494,6 @@ impl SwarmDriver { send_back_addr, } => { event_string = "incoming"; - // info!("{:?}", self.swarm.network_info()); trace!("IncomingConnection ({connection_id:?}) with local_addr: {local_addr:?} send_back_addr: {send_back_addr:?}"); } SwarmEvent::ConnectionEstablished { @@ -502,11 +501,11 @@ impl SwarmDriver { endpoint, num_established, connection_id, - .. + concurrent_dial_errors, + established_in, } => { event_string = "ConnectionEstablished"; - trace!(%peer_id, num_established, "ConnectionEstablished ({connection_id:?}): {}", endpoint_str(&endpoint)); - // info!(%peer_id, ?connection_id, "ConnectionEstablished {:?}", self.swarm.network_info()); + trace!(%peer_id, num_established, ?concurrent_dial_errors, "ConnectionEstablished ({connection_id:?}) in {established_in:?}: {}", endpoint_str(&endpoint)); let _ = self.live_connected_peers.insert( connection_id, @@ -525,7 +524,6 @@ impl SwarmDriver { connection_id, } => { event_string = "ConnectionClosed"; - // info!(%peer_id, ?connection_id, "ConnectionClosed: {:?}", self.swarm.network_info()); trace!(%peer_id, ?connection_id, ?cause, num_established, "ConnectionClosed: {}", endpoint_str(&endpoint)); let _ = self.live_connected_peers.remove(&connection_id); } @@ -669,7 +667,6 @@ impl SwarmDriver { send_back_addr, error, } => { - // info!("{:?}", self.swarm.network_info()); event_string = "Incoming ConnErr"; error!("IncomingConnectionError from local_addr:?{local_addr:?}, send_back_addr {send_back_addr:?} on {connection_id:?} with error {error:?}"); } @@ -683,11 +680,6 @@ impl SwarmDriver { SwarmEvent::NewExternalAddrCandidate { address } => { event_string = "NewExternalAddrCandidate"; - let all_external_addresses = self.swarm.external_addresses().collect_vec(); - let all_listeners = self.swarm.listeners().collect_vec(); - info!("All our listeners: {all_listeners:?}"); - info!("All our external addresses: {all_external_addresses:?}"); - if !self.swarm.external_addresses().any(|addr| addr == &address) && !self.is_client // If we are behind a home network, then our IP is returned here. We should be only having @@ -715,6 +707,10 @@ impl SwarmDriver { trace!("external address: listen port not set. This has to be set if you're running a node"); } } + let all_external_addresses = self.swarm.external_addresses().collect_vec(); + let all_listeners = self.swarm.listeners().collect_vec(); + debug!("All our listeners: {all_listeners:?}"); + debug!("All our external addresses: {all_external_addresses:?}"); } SwarmEvent::ExternalAddrConfirmed { address } => { event_string = "ExternalAddrConfirmed"; @@ -730,6 +726,7 @@ impl SwarmDriver { trace!("SwarmEvent has been ignored: {other:?}") } } + self.remove_outdated_connections(); self.log_handling(event_string.to_string(), start.elapsed()); @@ -1262,17 +1259,41 @@ impl SwarmDriver { } // Remove outdated connection to a peer if it is not in the RT. - fn _remove_outdated_connections(&mut self) { + fn remove_outdated_connections(&mut self) { let mut shall_removed = vec![]; - self.live_connected_peers - .retain(|connection_id, (peer_id, timeout)| { - let shall_retained = *timeout > Instant::now(); - if !shall_retained { - shall_removed.push((*connection_id, *peer_id)) + let timed_out_connections = + self.live_connected_peers + .iter() + .filter_map(|(connection_id, (peer_id, timeout))| { + if Instant::now() > *timeout { + Some((connection_id, peer_id)) + } else { + None + } + }); + + for (connection_id, peer_id) in timed_out_connections { + // Skip if the peer is present in our RT + if let Some(kbucket) = self.swarm.behaviour_mut().kademlia.kbucket(*peer_id) { + if kbucket + .iter() + .any(|peer_entry| *peer_id == *peer_entry.node.key.preimage()) + { + continue; } - shall_retained - }); + } + + // skip if the peer is a relay server that we're connected to + if self + .relay_manager + .is_peer_an_established_relay_server(peer_id) + { + continue; + } + + shall_removed.push((*connection_id, *peer_id)); + } if !shall_removed.is_empty() { trace!( @@ -1284,22 +1305,12 @@ impl SwarmDriver { shall_removed.len(), self.live_connected_peers.len() ); - } - // Only remove outdated peer not in the RT - for (connection_id, peer_id) in shall_removed { - if let Some(kbucket) = self.swarm.behaviour_mut().kademlia.kbucket(peer_id) { - if kbucket - .iter() - .any(|peer_entry| peer_id == *peer_entry.node.key.preimage()) - { - // Skip the connection as peer presents in the RT. - continue; - } + for (connection_id, peer_id) in shall_removed { + let _ = self.live_connected_peers.remove(&connection_id); + let result = self.swarm.close_connection(connection_id); + trace!("Removed outdated connection {connection_id:?} to {peer_id:?} with result: {result:?}"); } - - trace!("Removing outdated connection {connection_id:?} to {peer_id:?}"); - let _result = self.swarm.close_connection(connection_id); } } diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index 255512d377..5ae9fc509c 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -62,6 +62,10 @@ impl RelayManager { self.enabled = enable; } + pub(crate) fn is_peer_an_established_relay_server(&self, peer_id: &PeerId) -> bool { + self.connected_relays.contains_key(peer_id) + } + pub(crate) fn add_non_relayed_listener_id(&mut self, listener_id: ListenerId) { debug!("Adding non relayed listener id: {listener_id:?}"); self.non_relayed_listener_id.push_front(listener_id); From 3bcf527a7a5148df7bb0d7c457fadfc4db208e55 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 23 Apr 2024 19:54:31 +0530 Subject: [PATCH 126/205] fix: short circuit identify if the peer is already present in the routitng table --- sn_networking/src/event.rs | 89 +++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 39 deletions(-) diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 7223318394..8fa298b586 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -294,13 +294,13 @@ impl SwarmDriver { return Ok(()); } - let has_dialed = self.dialed_peers.contains(&peer_id); - // if client, return. if info.agent_version != IDENTIFY_NODE_VERSION_STR.to_string() { return Ok(()); } + let has_dialed = self.dialed_peers.contains(&peer_id); + // If we're not in local mode, only add globally reachable addresses. // Strip the `/p2p/...` part of the multiaddresses. // Collect into a HashSet directly to avoid multiple allocations and handle deduplication. @@ -329,46 +329,53 @@ impl SwarmDriver { // peer is external accessible, hence safe to be added into RT. if !self.local && !has_dialed { // Only need to dial back for not fulfilled kbucket - let (kbucket_full, ilog2) = if let Some(kbucket) = - self.swarm.behaviour_mut().kademlia.kbucket(peer_id) - { - let ilog2 = kbucket.range().0.ilog2(); - let num_peers = kbucket.num_entries(); - let mut is_bucket_full = num_peers >= K_VALUE.into(); - - // If the bucket contains any of a bootstrap node, - // consider the bucket is not full and dial back - // so that the bootstrap nodes can be replaced. - if is_bucket_full { - if let Some(peers) = self.bootstrap_peers.get(&ilog2) { - if kbucket - .iter() - .any(|entry| peers.contains(entry.node.key.preimage())) - { - is_bucket_full = false; + let (kbucket_full, already_present_in_rt, ilog2) = + if let Some(kbucket) = + self.swarm.behaviour_mut().kademlia.kbucket(peer_id) + { + let ilog2 = kbucket.range().0.ilog2(); + let num_peers = kbucket.num_entries(); + let mut is_bucket_full = num_peers >= K_VALUE.into(); + + // check if peer_id is already a part of RT + let already_present_in_rt = kbucket + .iter() + .any(|entry| entry.node.key.preimage() == &peer_id); + + // If the bucket contains any of a bootstrap node, + // consider the bucket is not full and dial back + // so that the bootstrap nodes can be replaced. + if is_bucket_full { + if let Some(peers) = self.bootstrap_peers.get(&ilog2) { + if kbucket.iter().any(|entry| { + peers.contains(entry.node.key.preimage()) + }) { + is_bucket_full = false; + } } } - } - (is_bucket_full, ilog2) - } else { - // Function will return `None` if the given key refers to self - // hence return true to skip further action. - (true, None) - }; - - if !kbucket_full { - info!(%peer_id, ?addrs, "received identify info from undialed peer for not full kbucket {ilog2:?}, dial back to confirm external accessible"); - if let Err(err) = self.swarm.dial( - DialOpts::peer_id(peer_id) - .condition(PeerCondition::NotDialing) - .addresses(addrs.iter().cloned().collect()) - .build(), - ) { - warn!(%peer_id, ?addrs, "dialing error: {err:?}"); - } - } else { - trace!("received identify for a full bucket {ilog2:?}, no dialing {peer_id:?} on {addrs:?}"); + (is_bucket_full, already_present_in_rt, ilog2) + } else { + return Ok(()); + }; + + if kbucket_full { + trace!("received identify for a full bucket {ilog2:?}, not dialing {peer_id:?} on {addrs:?}"); + return Ok(()); + } else if already_present_in_rt { + trace!("received identify for {peer_id:?} that is already part of the RT. Not dialing {peer_id:?} on {addrs:?}"); + return Ok(()); + } + + info!(%peer_id, ?addrs, "received identify info from undialed peer for not full kbucket {ilog2:?}, dial back to confirm external accessible"); + if let Err(err) = self.swarm.dial( + DialOpts::peer_id(peer_id) + .condition(PeerCondition::NotDialing) + .addresses(addrs.iter().cloned().collect()) + .build(), + ) { + warn!(%peer_id, ?addrs, "dialing error: {err:?}"); } trace!( @@ -409,6 +416,10 @@ impl SwarmDriver { } } } + trace!( + "SwarmEvent handled in {:?}: {event_string:?}", + start.elapsed() + ); } // Log the other Identify events. libp2p::identify::Event::Sent { .. } => trace!("identify: {iden:?}"), From 491db5312a551cbaa21bde84015b7339fa1e6f55 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 23 Apr 2024 20:12:16 +0530 Subject: [PATCH 127/205] fix: relay server should not close connections made to a reserved peer --- sn_networking/src/driver.rs | 3 +++ sn_networking/src/event.rs | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index 3bb5a08d10..ced249463e 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -539,6 +539,7 @@ impl NetworkBuilder { connected_peers: 0, bootstrap, relay_manager, + relay_server_reservations: Default::default(), close_group: Default::default(), replication_fetcher, #[cfg(feature = "open-metrics")] @@ -586,6 +587,8 @@ pub struct SwarmDriver { pub(crate) connected_peers: usize, pub(crate) bootstrap: ContinuousBootstrap, pub(crate) relay_manager: RelayManager, + /// The reservations given out by our relay server + pub(crate) relay_server_reservations: HashSet, /// The peers that are closer to our PeerId. Includes self. pub(crate) close_group: Vec, pub(crate) replication_fetcher: ReplicationFetcher, diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 8fa298b586..3fb6288b92 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -275,6 +275,19 @@ impl SwarmDriver { event_string = "relay_server_event"; info!(?event, "relay server event"); + + match *event { + libp2p::relay::Event::ReservationReqAccepted { + src_peer_id, + renewed: _, + } => { + self.relay_server_reservations.insert(src_peer_id); + } + libp2p::relay::Event::ReservationTimedOut { src_peer_id } => { + self.relay_server_reservations.remove(&src_peer_id); + } + _ => {} + } } SwarmEvent::Behaviour(NodeEvent::Identify(iden)) => { event_string = "identify"; @@ -1303,6 +1316,10 @@ impl SwarmDriver { continue; } + if self.relay_server_reservations.contains(peer_id) { + continue; + } + shall_removed.push((*connection_id, *peer_id)); } From d438197966b4d43f3d31d79f96547dcb65d3056d Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 24 Apr 2024 00:33:22 +0530 Subject: [PATCH 128/205] fix(relay_manager): remove external addr on connection close --- sn_networking/src/event.rs | 11 +++++------ sn_networking/src/relay_manager.rs | 23 ++++++++++++++++++----- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 3fb6288b92..b912936653 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -421,7 +421,7 @@ impl SwarmDriver { } if self.relay_manager.are_we_behind_nat(&mut self.swarm) { - error!("Node reported as being behind NAT for not having enough incoming connections. This can be false positive."); + warn!("Node reported as being behind NAT for not having enough incoming connections. This can be a false positive. Do nothing."); // self.send_event(NetworkEvent::TerminateNode { // reason: TerminateNodeReason::BehindNAT, @@ -510,7 +510,8 @@ impl SwarmDriver { } => { event_string = "listener closed"; info!("Listener {listener_id:?} with add {addresses:?} has been closed for {reason:?}"); - self.relay_manager.update_on_listener_closed(&listener_id); + self.relay_manager + .update_on_listener_closed(&listener_id, &mut self.swarm); } SwarmEvent::IncomingConnection { connection_id, @@ -1309,10 +1310,7 @@ impl SwarmDriver { } // skip if the peer is a relay server that we're connected to - if self - .relay_manager - .is_peer_an_established_relay_server(peer_id) - { + if self.relay_manager.keep_alive_peer(peer_id) { continue; } @@ -1333,6 +1331,7 @@ impl SwarmDriver { shall_removed.len(), self.live_connected_peers.len() ); + trace!(?self.relay_manager); for (connection_id, peer_id) in shall_removed { let _ = self.live_connected_peers.remove(&connection_id); diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index 5ae9fc509c..c4e2fd8994 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -18,6 +18,7 @@ const MAX_PEERS_IN_RT_DURING_NAT_CHECK: usize = 30; /// To manager relayed connections. // todo: try to dial whenever connected_relays drops below threshold. Need to perform this on interval. +#[derive(Debug)] pub(crate) struct RelayManager { // states enabled: bool, @@ -62,8 +63,9 @@ impl RelayManager { self.enabled = enable; } - pub(crate) fn is_peer_an_established_relay_server(&self, peer_id: &PeerId) -> bool { + pub(crate) fn keep_alive_peer(&self, peer_id: &PeerId) -> bool { self.connected_relays.contains_key(peer_id) + || self.waiting_for_reservation.contains_key(peer_id) } pub(crate) fn add_non_relayed_listener_id(&mut self, listener_id: ListenerId) { @@ -152,6 +154,13 @@ impl RelayManager { // todo: should we remove all our other `listen_addr`? And should we block from adding `add_external_address` if // we're behind nat? if let Some((peer_id, relay_addr)) = self.candidates.pop_front() { + if self.connected_relays.contains_key(&peer_id) + || self.waiting_for_reservation.contains_key(&peer_id) + { + trace!("We are already using {peer_id:?} as a relay server. Skipping."); + continue; + } + match swarm.listen_on(relay_addr.clone()) { Ok(id) => { info!("Sending reservation to relay {peer_id:?} on {relay_addr:?}"); @@ -164,7 +173,7 @@ impl RelayManager { } } } else { - debug!("No more relay candidates."); + trace!("No more relay candidates."); break; } } @@ -189,14 +198,18 @@ impl RelayManager { } /// Update our state if the reservation has been cancelled or if the relay has closed. - /// todo: remove from external addresses. - pub(crate) fn update_on_listener_closed(&mut self, listener_id: &ListenerId) { + pub(crate) fn update_on_listener_closed( + &mut self, + listener_id: &ListenerId, + swarm: &mut Swarm, + ) { let Some(peer_id) = self.relayed_listener_id_map.remove(listener_id) else { return; }; if let Some(addr) = self.connected_relays.remove(&peer_id) { - info!("Removed peer form connected_relays as the listener has been closed {peer_id:?}: {addr:?}"); + info!("Removed peer form connected_relays and external address as the listener has been closed {peer_id:?}: {addr:?}"); + swarm.remove_external_address(&addr); } else if let Some(addr) = self.waiting_for_reservation.remove(&peer_id) { info!("Removed peer form waiting_for_reservation as the listener has been closed {peer_id:?}: {addr:?}"); } else { From 7f15219bdacd14f704693ecd1c473eb5ab076712 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 24 Apr 2024 01:28:39 +0530 Subject: [PATCH 129/205] chore: remove non relayed listener id from relay manager --- sn_networking/src/driver.rs | 5 ++--- sn_networking/src/relay_manager.rs | 9 --------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index ced249463e..a2070a3daf 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -832,9 +832,8 @@ impl SwarmDriver { /// Listen on the provided address. Also records it within RelayManager pub(crate) fn listen_on(&mut self, addr: Multiaddr) -> Result<()> { - let id = self.swarm.listen_on(addr)?; - self.relay_manager.add_non_relayed_listener_id(id); - + let id = self.swarm.listen_on(addr.clone())?; + info!("Listening on {id:?} with addr: {addr:?}"); Ok(()) } } diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index c4e2fd8994..0f9e9d650f 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -28,9 +28,6 @@ pub(crate) struct RelayManager { /// Tracker for the relayed listen addresses. relayed_listener_id_map: HashMap, - /// Tracker for the non relayed listen addresses. These should be collected whenever we call `listen_on` from outside - /// the manager. - non_relayed_listener_id: VecDeque, } impl RelayManager { @@ -53,7 +50,6 @@ impl RelayManager { connected_relays: Default::default(), waiting_for_reservation: Default::default(), candidates, - non_relayed_listener_id: Default::default(), relayed_listener_id_map: Default::default(), } } @@ -68,11 +64,6 @@ impl RelayManager { || self.waiting_for_reservation.contains_key(peer_id) } - pub(crate) fn add_non_relayed_listener_id(&mut self, listener_id: ListenerId) { - debug!("Adding non relayed listener id: {listener_id:?}"); - self.non_relayed_listener_id.push_front(listener_id); - } - /// If we have 0 incoming connection even after we have a lot of peers, then we are behind a NAT pub(crate) fn are_we_behind_nat(&self, swarm: &mut Swarm) -> bool { if swarm From f1f0e098808c8075308c13ce650a620b4860881a Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 24 Apr 2024 01:55:10 +0530 Subject: [PATCH 130/205] chore: enable connection keepalive timeout --- sn_networking/src/driver.rs | 2 +- sn_networking/src/relay_manager.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index a2070a3daf..613d7fea10 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -93,7 +93,7 @@ const MAX_PACKET_SIZE: usize = 1024 * 1024 * 5; // the chunk size is 1mb, so sho // Timeout for requests sent/received through the request_response behaviour. const REQUEST_TIMEOUT_DEFAULT_S: Duration = Duration::from_secs(30); // Sets the keep-alive timeout of idle connections. -const CONNECTION_KEEP_ALIVE_TIMEOUT: Duration = Duration::from_secs(u64::MAX); +const CONNECTION_KEEP_ALIVE_TIMEOUT: Duration = Duration::from_secs(30); const NETWORKING_CHANNEL_SIZE: usize = 10_000; diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index 0f9e9d650f..91f3f73d40 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -17,7 +17,6 @@ const MAX_POTENTIAL_CANDIDATES: usize = 15; const MAX_PEERS_IN_RT_DURING_NAT_CHECK: usize = 30; /// To manager relayed connections. -// todo: try to dial whenever connected_relays drops below threshold. Need to perform this on interval. #[derive(Debug)] pub(crate) struct RelayManager { // states From f7beaa9f4282674371bfd37bdc148019abe3b229 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 24 Apr 2024 03:04:37 +0530 Subject: [PATCH 131/205] fix(ci): update memcheck bootstrap address rg --- .github/workflows/memcheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/memcheck.yml b/.github/workflows/memcheck.yml index 2239488c08..591e140c19 100644 --- a/.github/workflows/memcheck.yml +++ b/.github/workflows/memcheck.yml @@ -52,7 +52,7 @@ jobs: - name: Set SAFE_PEERS run: | - safe_peers=$(rg "listening on \".+\"" $BOOTSTRAP_NODE_DATA_PATH -u | \ + safe_peers=$(rg "Local node is listening .+ on \".+\"" $BOOTSTRAP_NODE_DATA_PATH -u | \ rg '/ip4.*$' -m1 -o | rg '"' -r '') echo "SAFE_PEERS=$safe_peers" >> $GITHUB_ENV From 2d350877ddd5244b3b23b2b0d2878025e4756795 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Wed, 24 Apr 2024 10:33:34 +0900 Subject: [PATCH 132/205] ci: increase token distribution build times by 5 mins --- .github/workflows/merge.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 820c506688..4039643392 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -604,7 +604,7 @@ jobs: - name: Build binaries run: cargo build --release --features=local-discovery,distribution --bin safenode --bin faucet - timeout-minutes: 30 + timeout-minutes: 35 - name: Build testing executable run: cargo test --release --features=local-discovery,distribution --no-run @@ -612,7 +612,7 @@ jobs: # only set the target dir for windows to bypass the linker issue. # happens if we build the node manager via testnet action CARGO_TARGET_DIR: ${{ matrix.os == 'windows-latest' && './test-target' || '.' }} - timeout-minutes: 30 + timeout-minutes: 35 - name: Start a local network uses: maidsafe/sn-local-testnet-action@main From c31a740fc131c5ede1510ad78bb4af851e1ba34a Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 24 Apr 2024 17:52:09 +0530 Subject: [PATCH 133/205] chore(ci): use apt-get instead of apt --- Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 23054ee34f..2b32a3a2ae 100644 --- a/Justfile +++ b/Justfile @@ -96,7 +96,7 @@ build-release-artifacts arch: if [[ "$(grep -E '^NAME="Ubuntu"' /etc/os-release)" ]]; then # This is intended for use on a fresh Github Actions agent sudo apt update -y - sudo apt install -y musl-tools + sudo apt-get install -y musl-tools fi rustup target add x86_64-unknown-linux-musl fi From 0415cf7263aa271043a885d3812867c16c30a964 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 24 Apr 2024 18:14:09 +0530 Subject: [PATCH 134/205] fix(network): remove all external addresses related to a relay server --- sn_networking/src/driver.rs | 2 +- sn_networking/src/relay_manager.rs | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index 613d7fea10..dfabc31666 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -524,7 +524,7 @@ impl NetworkBuilder { let bootstrap = ContinuousBootstrap::new(); let replication_fetcher = ReplicationFetcher::new(peer_id, network_event_sender.clone()); - let mut relay_manager = RelayManager::new(self.initial_peers); + let mut relay_manager = RelayManager::new(self.initial_peers, peer_id); if !is_client { relay_manager.enable_hole_punching(self.is_behind_home_network); } diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index 91f3f73d40..58bfcad00f 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -19,6 +19,7 @@ const MAX_PEERS_IN_RT_DURING_NAT_CHECK: usize = 30; /// To manager relayed connections. #[derive(Debug)] pub(crate) struct RelayManager { + self_peer_id: PeerId, // states enabled: bool, candidates: VecDeque<(PeerId, Multiaddr)>, @@ -30,7 +31,7 @@ pub(crate) struct RelayManager { } impl RelayManager { - pub(crate) fn new(initial_peers: Vec) -> Self { + pub(crate) fn new(initial_peers: Vec, self_peer_id: PeerId) -> Self { let candidates = initial_peers .into_iter() .filter_map(|addr| { @@ -45,6 +46,7 @@ impl RelayManager { }) .collect(); Self { + self_peer_id, enabled: false, connected_relays: Default::default(), waiting_for_reservation: Default::default(), @@ -198,8 +200,18 @@ impl RelayManager { }; if let Some(addr) = self.connected_relays.remove(&peer_id) { - info!("Removed peer form connected_relays and external address as the listener has been closed {peer_id:?}: {addr:?}"); + info!("Removing connected relay server as the listener has been closed: {peer_id:?}"); + info!("Removing external addr: {addr:?}"); swarm.remove_external_address(&addr); + + // Even though we craft and store addrs in this format /ip4/198.51.100.0/tcp/55555/p2p/QmRelay/p2p-circuit/, + // sometimes our PeerId is added at the end by the swarm?, which we want to remove as well i.e., + // /ip4/198.51.100.0/tcp/55555/p2p/QmRelay/p2p-circuit/p2p/QmSelf + let Ok(addr_with_self_peer_id) = addr.with_p2p(self.self_peer_id) else { + return; + }; + info!("Removing external addr: {addr_with_self_peer_id:?}"); + swarm.remove_external_address(&addr_with_self_peer_id); } else if let Some(addr) = self.waiting_for_reservation.remove(&peer_id) { info!("Removed peer form waiting_for_reservation as the listener has been closed {peer_id:?}: {addr:?}"); } else { @@ -216,7 +228,7 @@ impl RelayManager { false } - // the listen addr should be something like `/ip4/198.51.100.0/tcp/55555/p2p/QmRelay/p2p-circuit/ + /// The listen addr should be something like /ip4/198.51.100.0/tcp/55555/p2p/QmRelay/p2p-circuit/ fn craft_relay_address(addr: &Multiaddr, peer_id: Option) -> Option { let mut output_addr = Multiaddr::empty(); From 362c9a3d31601dc2eeb38e3c376301fb26a5b5e1 Mon Sep 17 00:00:00 2001 From: qima Date: Wed, 24 Apr 2024 18:17:38 +0800 Subject: [PATCH 135/205] chore(CI): upload faucet log during CI --- .github/workflows/memcheck.yml | 22 ++++++++++++++++++++++ sn_cli/src/bin/main.rs | 10 ++++++++-- sn_faucet/src/main.rs | 9 ++++++++- sn_node/src/bin/safenode/main.rs | 5 ++++- 4 files changed, 42 insertions(+), 4 deletions(-) diff --git a/.github/workflows/memcheck.yml b/.github/workflows/memcheck.yml index 591e140c19..4403dc57e8 100644 --- a/.github/workflows/memcheck.yml +++ b/.github/workflows/memcheck.yml @@ -15,6 +15,7 @@ env: NODE_DATA_PATH: /home/runner/.local/share/safe/node BOOTSTRAP_NODE_DATA_PATH: /home/runner/.local/share/safe/bootstrap_node RESTART_TEST_NODE_DATA_PATH: /home/runner/.local/share/safe/restart_node + FAUCET_LOG_PATH: /home/runner/.local/share/safe/test_faucet/logs jobs: memory-check: @@ -105,6 +106,19 @@ jobs: SN_LOG: "all" timeout-minutes: 15 + - name: Move faucet log to the working folder + run: | + echo "SAFE_DATA_PATH has: " + ls -l $SAFE_DATA_PATH + echo "test_faucet foder has: " + ls -l $SAFE_DATA_PATH/test_faucet + echo "logs folder has: " + ls -l $SAFE_DATA_PATH/test_faucet/logs + mv $FAUCET_LOG_PATH/faucet.log ./faucet_log.log + continue-on-error: true + if: always() + timeout-minutes: 1 + - name: Download 95mb file to be uploaded with the safe client shell: bash run: wget https://sn-node.s3.eu-west-2.amazonaws.com/the-test-data.zip @@ -397,3 +411,11 @@ jobs: path: initial_balance_from_faucet.txt continue-on-error: true if: always() + + - name: Upload faucet log + uses: actions/upload-artifact@main + with: + name: memory_check_faucet_log + path: faucet_log.log + continue-on-error: true + if: always() diff --git a/sn_cli/src/bin/main.rs b/sn_cli/src/bin/main.rs index 62bb8fafc0..2aa0309041 100644 --- a/sn_cli/src/bin/main.rs +++ b/sn_cli/src/bin/main.rs @@ -64,8 +64,14 @@ async fn main() -> Result<()> { // Log the full command that was run info!("\"{}\"", std::env::args().collect::>().join(" ")); - debug!("Built with git version: {}", sn_build_info::git_info()); - println!("Built with git version: {}", sn_build_info::git_info()); + debug!( + "safe client built with git version: {}", + sn_build_info::git_info() + ); + println!( + "safe client built with git version: {}", + sn_build_info::git_info() + ); let client_data_dir_path = get_client_data_dir_path()?; // Perform actions that do not require us connecting to the network and return early diff --git a/sn_faucet/src/main.rs b/sn_faucet/src/main.rs index 07ac8d54a2..639c37270b 100644 --- a/sn_faucet/src/main.rs +++ b/sn_faucet/src/main.rs @@ -56,7 +56,14 @@ async fn main() -> Result<()> { log_builder.output_dest(opt.log_output_dest); let _log_handles = log_builder.initialize()?; - debug!("Built with git version: {}", sn_build_info::git_info()); + debug!( + "faucet built with git version: {}", + sn_build_info::git_info() + ); + println!( + "faucet built with git version: {}", + sn_build_info::git_info() + ); info!("Instantiating a SAFE Test Faucet..."); let secret_key = bls::SecretKey::random(); diff --git a/sn_node/src/bin/safenode/main.rs b/sn_node/src/bin/safenode/main.rs index 56b337ce3c..9ff693dc46 100644 --- a/sn_node/src/bin/safenode/main.rs +++ b/sn_node/src/bin/safenode/main.rs @@ -176,7 +176,10 @@ fn main() -> Result<()> { env!("CARGO_PKG_VERSION") ); info!("\n{}\n{}", msg, "=".repeat(msg.len())); - debug!("Built with git version: {}", sn_build_info::git_info()); + debug!( + "safenode built with git version: {}", + sn_build_info::git_info() + ); info!("Node started with initial_peers {bootstrap_peers:?}"); From 04fbc761eb82257b90b4052e1ab525a1b071b074 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 24 Apr 2024 22:16:45 +0530 Subject: [PATCH 136/205] chore: spelling fix This commit is to just trigger a new release --- sn_networking/src/driver.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index dfabc31666..a4141436c8 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -641,7 +641,7 @@ impl SwarmDriver { tokio::select! { swarm_event = self.swarm.select_next_some() => { // logging for handling events happens inside handle_swarm_events - // otherwise we're rewriting match statements etc around this anwyay + // otherwise we're rewriting match statements etc around this anyway if let Err(err) = self.handle_swarm_events(swarm_event) { warn!("Error while handling swarm event: {err}"); } From 15b26903ee96e30209b3861e7263d5448caa44dc Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 24 Apr 2024 16:51:20 +0000 Subject: [PATCH 137/205] chore(release): sn_client-v0.106.0-alpha.4/sn_networking-v0.15.0-alpha.4/sn_cli-v0.91.0-alpha.4/sn_node-v0.106.0-alpha.4/sn_auditor-v0.1.11-alpha.0/sn_faucet-v0.4.13-alpha.0/sn_node_rpc_client-v0.6.12-alpha.0 --- Cargo.lock | 14 +++++++------- sn_auditor/Cargo.toml | 4 ++-- sn_cli/Cargo.toml | 6 +++--- sn_client/Cargo.toml | 4 ++-- sn_faucet/Cargo.toml | 4 ++-- sn_networking/Cargo.toml | 2 +- sn_node/Cargo.toml | 6 +++--- sn_node_rpc_client/Cargo.toml | 6 +++--- 8 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1b32f04e3d..7548fe3e91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5322,7 +5322,7 @@ dependencies = [ [[package]] name = "sn_auditor" -version = "0.1.10-alpha.0" +version = "0.1.11-alpha.0" dependencies = [ "blsttc", "clap", @@ -5348,7 +5348,7 @@ dependencies = [ [[package]] name = "sn_cli" -version = "0.91.0-alpha.3" +version = "0.91.0-alpha.4" dependencies = [ "aes 0.7.5", "base64 0.22.0", @@ -5389,7 +5389,7 @@ dependencies = [ [[package]] name = "sn_client" -version = "0.106.0-alpha.3" +version = "0.106.0-alpha.4" dependencies = [ "assert_matches", "async-trait", @@ -5438,7 +5438,7 @@ dependencies = [ [[package]] name = "sn_faucet" -version = "0.4.12-alpha.0" +version = "0.4.13-alpha.0" dependencies = [ "assert_fs", "base64 0.22.0", @@ -5505,7 +5505,7 @@ dependencies = [ [[package]] name = "sn_networking" -version = "0.15.0-alpha.3" +version = "0.15.0-alpha.4" dependencies = [ "aes-gcm-siv", "async-trait", @@ -5545,7 +5545,7 @@ dependencies = [ [[package]] name = "sn_node" -version = "0.106.0-alpha.3" +version = "0.106.0-alpha.4" dependencies = [ "assert_fs", "assert_matches", @@ -5600,7 +5600,7 @@ dependencies = [ [[package]] name = "sn_node_rpc_client" -version = "0.6.11-alpha.0" +version = "0.6.12-alpha.0" dependencies = [ "assert_fs", "async-trait", diff --git a/sn_auditor/Cargo.toml b/sn_auditor/Cargo.toml index 1738195dc4..d88446cbeb 100644 --- a/sn_auditor/Cargo.toml +++ b/sn_auditor/Cargo.toml @@ -2,7 +2,7 @@ authors = ["MaidSafe Developers "] description = "Safe Network Auditor" name = "sn_auditor" -version = "0.1.10-alpha.0" +version = "0.1.11-alpha.0" edition = "2021" homepage = "https://maidsafe.net" repository = "https://github.com/maidsafe/safe_network" @@ -24,7 +24,7 @@ dirs-next = "~2.0.0" graphviz-rust = "0.9.0" serde = { version = "1.0.133", features = [ "derive", "rc" ]} serde_json = "1.0.108" -sn_client = { path = "../sn_client", version = "0.106.0-alpha.3" } +sn_client = { path = "../sn_client", version = "0.106.0-alpha.4" } sn_logging = { path = "../sn_logging", version = "0.2.26-alpha.0" } sn_peers_acquisition= { path="../sn_peers_acquisition", version = "0.2.11-alpha.0" } tiny_http = { version="0.12", features = ["ssl-rustls"] } diff --git a/sn_cli/Cargo.toml b/sn_cli/Cargo.toml index 06907c76cc..13cbeccc12 100644 --- a/sn_cli/Cargo.toml +++ b/sn_cli/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_cli" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.91.0-alpha.3" +version = "0.91.0-alpha.4" [[bin]] path="src/bin/main.rs" @@ -53,7 +53,7 @@ reqwest = { version="0.12.2", default-features=false, features = ["rustls-tls-ma rmp-serde = "1.1.1" serde = { version = "1.0.133", features = [ "derive"]} sn_build_info = { path="../sn_build_info", version = "0.1.7" } -sn_client = { path = "../sn_client", version = "0.106.0-alpha.3" } +sn_client = { path = "../sn_client", version = "0.106.0-alpha.4" } sn_logging = { path = "../sn_logging", version = "0.2.26-alpha.0" } sn_peers_acquisition= { path="../sn_peers_acquisition", version = "0.2.11-alpha.0" } sn_protocol = { path = "../sn_protocol", version = "0.16.4-alpha.0" } @@ -70,7 +70,7 @@ eyre = "0.6.8" criterion = "0.5.1" tempfile = "3.6.0" rand = { version = "~0.8.5", features = ["small_rng"] } -sn_client = { path = "../sn_client", version = "0.106.0-alpha.3", features = ["test-utils"] } +sn_client = { path = "../sn_client", version = "0.106.0-alpha.4", features = ["test-utils"] } [lints] workspace = true diff --git a/sn_client/Cargo.toml b/sn_client/Cargo.toml index 8920348718..64afbb6bf7 100644 --- a/sn_client/Cargo.toml +++ b/sn_client/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_client" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.106.0-alpha.3" +version = "0.106.0-alpha.4" [features] default=[] @@ -36,7 +36,7 @@ rayon = "1.8.0" rmp-serde = "1.1.1" self_encryption = "~0.29.0" serde = { version = "1.0.133", features = [ "derive", "rc" ]} -sn_networking = { path = "../sn_networking", version = "0.15.0-alpha.3" } +sn_networking = { path = "../sn_networking", version = "0.15.0-alpha.4" } sn_protocol = { path = "../sn_protocol", version = "0.16.4-alpha.0" } serde_json = "1.0" sn_registers = { path = "../sn_registers", version = "0.3.13-alpha.0" } diff --git a/sn_faucet/Cargo.toml b/sn_faucet/Cargo.toml index 5223041530..944731d5d3 100644 --- a/sn_faucet/Cargo.toml +++ b/sn_faucet/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_faucet" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.4.12-alpha.0" +version = "0.4.13-alpha.0" [features] default = [] @@ -36,7 +36,7 @@ minreq = { version = "2.11.0", features = ["https-rustls"], optional = true } serde = { version = "1.0.193", features = ["derive"] } serde_json = "1.0.108" sn_build_info = { path = "../sn_build_info", version = "0.1.7" } -sn_client = { path = "../sn_client", version = "0.106.0-alpha.3" } +sn_client = { path = "../sn_client", version = "0.106.0-alpha.4" } sn_logging = { path = "../sn_logging", version = "0.2.26-alpha.0" } sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.11-alpha.0" } sn_transfers = { path = "../sn_transfers", version = "0.18.0-alpha.0" } diff --git a/sn_networking/Cargo.toml b/sn_networking/Cargo.toml index c0b493cc07..8ce2b5c437 100644 --- a/sn_networking/Cargo.toml +++ b/sn_networking/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_networking" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.15.0-alpha.3" +version = "0.15.0-alpha.4" [features] default = ["libp2p/quic"] diff --git a/sn_node/Cargo.toml b/sn_node/Cargo.toml index 123d6578b7..34038c8347 100644 --- a/sn_node/Cargo.toml +++ b/sn_node/Cargo.toml @@ -2,7 +2,7 @@ authors = ["MaidSafe Developers "] description = "Safe Node" name = "sn_node" -version = "0.106.0-alpha.3" +version = "0.106.0-alpha.4" edition = "2021" license = "GPL-3.0" homepage = "https://maidsafe.net" @@ -51,9 +51,9 @@ self_encryption = "~0.29.0" serde = { version = "1.0.133", features = ["derive", "rc"] } sn_build_info = { path = "../sn_build_info", version = "0.1.7" } sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.11-alpha.0" } -sn_client = { path = "../sn_client", version = "0.106.0-alpha.3" } +sn_client = { path = "../sn_client", version = "0.106.0-alpha.4" } sn_logging = { path = "../sn_logging", version = "0.2.26-alpha.0" } -sn_networking = { path = "../sn_networking", version = "0.15.0-alpha.3" } +sn_networking = { path = "../sn_networking", version = "0.15.0-alpha.4" } sn_protocol = { path = "../sn_protocol", version = "0.16.4-alpha.0" } sn_registers = { path = "../sn_registers", version = "0.3.13-alpha.0" } sn_transfers = { path = "../sn_transfers", version = "0.18.0-alpha.0" } diff --git a/sn_node_rpc_client/Cargo.toml b/sn_node_rpc_client/Cargo.toml index d849c05f97..71676764bc 100644 --- a/sn_node_rpc_client/Cargo.toml +++ b/sn_node_rpc_client/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_node_rpc_client" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.6.11-alpha.0" +version = "0.6.12-alpha.0" [[bin]] name = "safenode_rpc_client" @@ -23,9 +23,9 @@ color-eyre = "0.6.2" hex = "~0.4.3" libp2p = { version="0.53", features = ["kad"]} libp2p-identity = { version="0.2.7", features = ["rand"] } -sn_client = { path = "../sn_client", version = "0.106.0-alpha.3" } +sn_client = { path = "../sn_client", version = "0.106.0-alpha.4" } sn_logging = { path = "../sn_logging", version = "0.2.26-alpha.0" } -sn_node = { path = "../sn_node", version = "0.106.0-alpha.3" } +sn_node = { path = "../sn_node", version = "0.106.0-alpha.4" } sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.11-alpha.0" } sn_protocol = { path = "../sn_protocol", version = "0.16.4-alpha.0", features=["rpc"] } sn_service_management = { path = "../sn_service_management", version = "0.2.5-alpha.0" } From d501e3b4c12d44d0aa277b99a49f64a94def7f81 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 24 Apr 2024 18:45:38 +0100 Subject: [PATCH 138/205] fix: do not delete custom bin on `add` cmd When the `--path` argument is specified on the node manager's `add` command, a custom binary is supplied, and now, the source of `--path` is not deleted. By default, the `add` command downloads the latest version of a binary to a temporary location, and that binary is then copied to the service location. Having finished with it, the node manager then cleans up the temporary binary. When the `--path` argument is used, it goes through the same code path, and hence the source of `--path` was being deleted as a side effect. Now, a flag is used to indicate whether the binary should be deleted, and it is set to `false` when the `--path` argument is supplied. The `--path` argument is being used when users build their own `safenode`, so they are expecting that the built binary will still exist. --- sn_node_manager/src/add_services/config.rs | 1 + sn_node_manager/src/add_services/mod.rs | 4 +- sn_node_manager/src/add_services/tests.rs | 103 +++++++++++++++++++++ sn_node_manager/src/cmd/node.rs | 3 +- 4 files changed, 109 insertions(+), 2 deletions(-) diff --git a/sn_node_manager/src/add_services/config.rs b/sn_node_manager/src/add_services/config.rs index ed1855f2a9..70ab0dd713 100644 --- a/sn_node_manager/src/add_services/config.rs +++ b/sn_node_manager/src/add_services/config.rs @@ -108,6 +108,7 @@ impl InstallNodeServiceCtxBuilder { pub struct AddNodeServiceOptions { pub bootstrap_peers: Vec, pub count: Option, + pub delete_safenode_src: bool, pub env_variables: Option>, pub genesis: bool, pub local: bool, diff --git a/sn_node_manager/src/add_services/mod.rs b/sn_node_manager/src/add_services/mod.rs index 37f265e311..2e0232311d 100644 --- a/sn_node_manager/src/add_services/mod.rs +++ b/sn_node_manager/src/add_services/mod.rs @@ -198,7 +198,9 @@ pub async fn add_node( rpc_port = increment_port_option(rpc_port); } - std::fs::remove_file(options.safenode_src_path)?; + if options.delete_safenode_src { + std::fs::remove_file(options.safenode_src_path)?; + } if !added_service_data.is_empty() { println!("Services Added:"); diff --git a/sn_node_manager/src/add_services/tests.rs b/sn_node_manager/src/add_services/tests.rs index b8497b68d2..3e3ac0669a 100644 --- a/sn_node_manager/src/add_services/tests.rs +++ b/sn_node_manager/src/add_services/tests.rs @@ -133,6 +133,7 @@ async fn add_genesis_node_should_use_latest_version_and_add_one_service() -> Res AddNodeServiceOptions { bootstrap_peers: vec![], count: None, + delete_safenode_src: true, env_variables: None, genesis: true, local: true, @@ -229,6 +230,7 @@ async fn add_genesis_node_should_return_an_error_if_there_is_already_a_genesis_n AddNodeServiceOptions { bootstrap_peers: vec![], count: None, + delete_safenode_src: true, env_variables: None, genesis: true, local: true, @@ -286,6 +288,7 @@ async fn add_genesis_node_should_return_an_error_if_count_is_greater_than_1() -> AddNodeServiceOptions { bootstrap_peers: vec![], count: Some(3), + delete_safenode_src: true, env_variables: None, genesis: true, local: true, @@ -442,6 +445,7 @@ async fn add_node_should_use_latest_version_and_add_three_services() -> Result<( AddNodeServiceOptions { bootstrap_peers: vec![], count: Some(3), + delete_safenode_src: true, env_variables: None, genesis: false, local: false, @@ -582,6 +586,7 @@ async fn add_node_should_update_the_bootstrap_peers_inside_node_registry() -> Re AddNodeServiceOptions { bootstrap_peers: new_peers.clone(), count: None, + delete_safenode_src: true, env_variables: None, local: false, genesis: false, @@ -696,6 +701,7 @@ async fn add_node_should_update_the_environment_variables_inside_node_registry() AddNodeServiceOptions { bootstrap_peers: vec![], count: None, + delete_safenode_src: true, env_variables: env_variables.clone(), genesis: false, local: false, @@ -820,6 +826,7 @@ async fn add_new_node_should_add_another_service() -> Result<()> { AddNodeServiceOptions { bootstrap_peers: vec![], count: None, + delete_safenode_src: true, env_variables: None, genesis: false, local: false, @@ -925,6 +932,7 @@ async fn add_node_should_use_custom_ports_for_one_service() -> Result<()> { AddNodeServiceOptions { bootstrap_peers: vec![], count: None, + delete_safenode_src: true, env_variables: None, genesis: false, local: false, @@ -1136,6 +1144,7 @@ async fn add_node_should_use_a_custom_port_range() -> Result<()> { AddNodeServiceOptions { bootstrap_peers: vec![], count: Some(3), + delete_safenode_src: true, env_variables: None, genesis: false, local: false, @@ -1190,6 +1199,7 @@ async fn add_node_should_return_an_error_if_port_and_node_count_do_not_match() - AddNodeServiceOptions { bootstrap_peers: vec![], count: Some(2), + delete_safenode_src: true, env_variables: None, genesis: false, local: false, @@ -1250,6 +1260,7 @@ async fn add_node_should_return_an_error_if_multiple_services_are_specified_with AddNodeServiceOptions { bootstrap_peers: vec![], count: Some(2), + delete_safenode_src: true, env_variables: None, genesis: false, local: false, @@ -1448,6 +1459,7 @@ async fn add_node_should_use_a_custom_port_range_for_metrics_server() -> Result< AddNodeServiceOptions { bootstrap_peers: vec![], count: Some(3), + delete_safenode_src: true, env_variables: None, genesis: false, local: false, @@ -1620,6 +1632,7 @@ async fn add_node_should_use_a_custom_port_range_for_the_rpc_server() -> Result< AddNodeServiceOptions { bootstrap_peers: vec![], count: Some(3), + delete_safenode_src: true, env_variables: None, genesis: false, local: false, @@ -1936,3 +1949,93 @@ async fn add_daemon_should_return_an_error_if_a_daemon_service_was_already_creat Ok(()) } + +#[tokio::test] +async fn add_node_should_not_delete_the_source_binary_if_path_arg_is_used() -> Result<()> { + let tmp_data_dir = assert_fs::TempDir::new()?; + let node_reg_path = tmp_data_dir.child("node_reg.json"); + + let mut mock_service_control = MockServiceControl::new(); + + let mut node_registry = NodeRegistry { + faucet: None, + save_path: node_reg_path.to_path_buf(), + nodes: vec![], + bootstrap_peers: vec![], + environment_variables: None, + daemon: None, + }; + + let latest_version = "0.96.4"; + let temp_dir = assert_fs::TempDir::new()?; + let node_data_dir = temp_dir.child("data"); + node_data_dir.create_dir_all()?; + let node_logs_dir = temp_dir.child("logs"); + node_logs_dir.create_dir_all()?; + let safenode_download_path = temp_dir.child(SAFENODE_FILE_NAME); + safenode_download_path.write_binary(b"fake safenode bin")?; + + let mut seq = Sequence::new(); + + // Expected calls for first installation + mock_service_control + .expect_get_available_port() + .times(1) + .returning(|| Ok(8081)) + .in_sequence(&mut seq); + + let install_ctx = InstallNodeServiceCtxBuilder { + bootstrap_peers: vec![], + data_dir_path: node_data_dir.to_path_buf().join("safenode1"), + env_variables: None, + genesis: false, + local: false, + log_dir_path: node_logs_dir.to_path_buf().join("safenode1"), + metrics_port: None, + name: "safenode1".to_string(), + node_port: None, + rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), + safenode_path: node_data_dir + .to_path_buf() + .join("safenode1") + .join(SAFENODE_FILE_NAME), + service_user: get_username(), + } + .build()?; + + mock_service_control + .expect_install() + .times(1) + .with(eq(install_ctx)) + .returning(|_| Ok(())) + .in_sequence(&mut seq); + + add_node( + AddNodeServiceOptions { + bootstrap_peers: vec![], + count: Some(1), + delete_safenode_src: false, + env_variables: None, + genesis: false, + local: false, + metrics_port: None, + node_port: None, + rpc_address: None, + rpc_port: None, + safenode_dir_path: temp_dir.to_path_buf(), + safenode_src_path: safenode_download_path.to_path_buf(), + service_data_dir_path: node_data_dir.to_path_buf(), + service_log_dir_path: node_logs_dir.to_path_buf(), + user: get_username(), + version: latest_version.to_string(), + }, + &mut node_registry, + &mock_service_control, + VerbosityLevel::Normal, + ) + .await?; + + safenode_download_path.assert(predicate::path::is_file()); + + Ok(()) +} diff --git a/sn_node_manager/src/cmd/node.rs b/sn_node_manager/src/cmd/node.rs index 52091890f1..e2deac217c 100644 --- a/sn_node_manager/src/cmd/node.rs +++ b/sn_node_manager/src/cmd/node.rs @@ -73,7 +73,7 @@ pub async fn add( let mut node_registry = NodeRegistry::load(&config::get_node_registry_path()?)?; let release_repo = ::default_config(); - let (safenode_src_path, version) = if let Some(path) = src_path { + let (safenode_src_path, version) = if let Some(path) = src_path.clone() { let version = get_bin_version(&path)?; (path, version) } else { @@ -102,6 +102,7 @@ pub async fn add( let options = AddNodeServiceOptions { count, + delete_safenode_src: src_path.is_none(), env_variables, genesis: is_first, local, From 2c5209b0f2a53ec1aaa703e5877df92b0f16b24a Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 25 Apr 2024 11:50:51 +0900 Subject: [PATCH 139/205] feat(client): speed up register checks when paying We take the existence of Payment at regsiter.sync as a proxy for intent to create a register. With that we can effectively use RetryStrategy::Quick under the hood to speed up initial register PUTs (15s check vs 60s with retries). --- sn_client/src/api.rs | 34 ++++++++++++++++++++++++++++++++++ sn_client/src/register.rs | 13 ++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/sn_client/src/api.rs b/sn_client/src/api.rs index 626c3c389f..11a5cf15fb 100644 --- a/sn_client/src/api.rs +++ b/sn_client/src/api.rs @@ -813,6 +813,40 @@ impl Client { self.get_signed_register_from_network(address, true).await } + /// Quickly checks if a `Register` is stored by expected nodes on the network. + /// + /// To be used for initial register put checks eg, if we expect the data _not_ + /// to exist, we can use it and essentially use the RetryStrategy::Quick under the hood + /// + /// + /// # Example + /// ```no_run + /// use sn_client::{Client, Error}; + /// use bls::SecretKey; + /// use xor_name::XorName; + /// use sn_registers::RegisterAddress; + /// # #[tokio::main] + /// # async fn main() -> Result<(),Error>{ + /// // Set up Client + /// let client = Client::new(SecretKey::random(), None, None, None).await?; + /// // Set up an address + /// let mut rng = rand::thread_rng(); + /// let owner = SecretKey::random().public_key(); + /// let xorname = XorName::random(&mut rng); + /// let address = RegisterAddress::new(xorname, owner); + /// // Verify address is stored + /// let is_stored = client.verify_register_stored(address).await.is_ok(); + /// # Ok(()) + /// # } + /// ``` + pub async fn quickly_check_if_register_stored( + &self, + address: RegisterAddress, + ) -> Result { + info!("Quickly checking for existing register : {address:?}"); + self.get_signed_register_from_network(address, false).await + } + /// Send a `SpendCashNote` request to the network. Protected method. /// /// # Arguments diff --git a/sn_client/src/register.rs b/sn_client/src/register.rs index da0ca39e8a..60b410b68f 100644 --- a/sn_client/src/register.rs +++ b/sn_client/src/register.rs @@ -500,7 +500,18 @@ impl ClientRegister { let mut royalties_fees = NanoTokens::zero(); let reg_result = if verify_store { debug!("VERIFYING REGISTER STORED {:?}", self.address()); - let res = self.client.verify_register_stored(*self.address()).await; + + let res = if payment_info.is_some() { + // we expect this to be a _fresh_ register. + // It still could have been PUT previously, but we'll do a quick verification + // instead of thorough one. + self.client + .quickly_check_if_register_stored(*self.address()) + .await + } else { + self.client.verify_register_stored(*self.address()).await + }; + // we need to keep the error here if verifying, so we can retry and pay for storage // once more below match res { From ac77ad7951f11412d5d3335654d5e846c990d271 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Thu, 25 Apr 2024 17:22:57 +0530 Subject: [PATCH 140/205] chore(network): remove nat detection via incoming connections check --- sn_networking/src/event.rs | 16 +++++++--------- sn_networking/src/relay_manager.rs | 25 ------------------------- 2 files changed, 7 insertions(+), 34 deletions(-) diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index b912936653..31e04f19c9 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -163,7 +163,6 @@ pub enum NetworkEvent { #[derive(Debug, Clone)] pub enum TerminateNodeReason { HardDiskWriteError, - BehindNAT, } // Manually implement Debug as `#[debug(with = "unverified_record_fmt")]` not working as expected. @@ -257,6 +256,13 @@ impl SwarmDriver { event_string = "kad_event"; self.handle_kad_event(kad_event)?; } + SwarmEvent::Behaviour(NodeEvent::Dcutr(event)) => { + event_string = "dcutr_event"; + info!( + "Dcutr with remote peer: {:?} is: {:?}", + event.remote_peer_id, event.result + ); + } SwarmEvent::Behaviour(NodeEvent::RelayClient(event)) => { event_string = "relay_client_event"; @@ -419,14 +425,6 @@ impl SwarmDriver { .kademlia .add_address(&peer_id, multiaddr.clone()); } - - if self.relay_manager.are_we_behind_nat(&mut self.swarm) { - warn!("Node reported as being behind NAT for not having enough incoming connections. This can be a false positive. Do nothing."); - - // self.send_event(NetworkEvent::TerminateNode { - // reason: TerminateNodeReason::BehindNAT, - // }) - } } } trace!( diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index 58bfcad00f..720a1fc1cf 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -14,7 +14,6 @@ use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; const MAX_CONCURRENT_RELAY_CONNECTIONS: usize = 3; const MAX_POTENTIAL_CANDIDATES: usize = 15; -const MAX_PEERS_IN_RT_DURING_NAT_CHECK: usize = 30; /// To manager relayed connections. #[derive(Debug)] @@ -65,30 +64,6 @@ impl RelayManager { || self.waiting_for_reservation.contains_key(peer_id) } - /// If we have 0 incoming connection even after we have a lot of peers, then we are behind a NAT - pub(crate) fn are_we_behind_nat(&self, swarm: &mut Swarm) -> bool { - if swarm - .network_info() - .connection_counters() - .num_established_incoming() - == 0 - || swarm - .network_info() - .connection_counters() - .num_pending_incoming() - == 0 - { - let mut total_peers = 0; - for kbucket in swarm.behaviour_mut().kademlia.kbuckets() { - total_peers += kbucket.num_entries(); - if total_peers > MAX_PEERS_IN_RT_DURING_NAT_CHECK { - return true; - } - } - } - false - } - /// Add a potential candidate to the list if it satisfies all the identify checks and also supports the relay server /// protocol. pub(crate) fn add_potential_candidates( From e44f46dfc83590f8a539982249b0a00767fe96e0 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Thu, 25 Apr 2024 15:41:19 +0100 Subject: [PATCH 141/205] feat: provide `--home-network` arg for `add` cmd The `add` command now enables the user to pass the `--home-network` argument to the `safenode` service. BREAKING CHANGE: the addition of the `home_network` field to `NodeServiceData` will mean this version of the node manager will not be able to deserialize old node registry files. --- sn_node_manager/src/add_services/config.rs | 5 + sn_node_manager/src/add_services/mod.rs | 2 + sn_node_manager/src/add_services/tests.rs | 153 ++++++++++++++++++--- sn_node_manager/src/bin/cli/main.rs | 7 + sn_node_manager/src/cmd/node.rs | 2 + sn_node_manager/src/lib.rs | 88 +++++++----- sn_node_manager/src/local.rs | 2 +- sn_node_manager/src/rpc.rs | 35 ++--- sn_service_management/src/node.rs | 1 + 9 files changed, 224 insertions(+), 71 deletions(-) diff --git a/sn_node_manager/src/add_services/config.rs b/sn_node_manager/src/add_services/config.rs index 70ab0dd713..fbfa9a696c 100644 --- a/sn_node_manager/src/add_services/config.rs +++ b/sn_node_manager/src/add_services/config.rs @@ -45,6 +45,7 @@ pub struct InstallNodeServiceCtxBuilder { pub data_dir_path: PathBuf, pub env_variables: Option>, pub genesis: bool, + pub home_network: bool, pub local: bool, pub log_dir_path: PathBuf, pub name: String, @@ -70,6 +71,9 @@ impl InstallNodeServiceCtxBuilder { if self.genesis { args.push(OsString::from("--first")); } + if self.home_network { + args.push(OsString::from("--home-network")); + } if self.local { args.push(OsString::from("--local")); } @@ -111,6 +115,7 @@ pub struct AddNodeServiceOptions { pub delete_safenode_src: bool, pub env_variables: Option>, pub genesis: bool, + pub home_network: bool, pub local: bool, pub metrics_port: Option, pub node_port: Option, diff --git a/sn_node_manager/src/add_services/mod.rs b/sn_node_manager/src/add_services/mod.rs index 2e0232311d..9192124eb8 100644 --- a/sn_node_manager/src/add_services/mod.rs +++ b/sn_node_manager/src/add_services/mod.rs @@ -145,6 +145,7 @@ pub async fn add_node( data_dir_path: service_data_dir_path.clone(), env_variables: options.env_variables.clone(), genesis: options.genesis, + home_network: options.home_network, local: options.local, log_dir_path: service_log_dir_path.clone(), metrics_port, @@ -170,6 +171,7 @@ pub async fn add_node( connected_peers: None, data_dir_path: service_data_dir_path.clone(), genesis: options.genesis, + home_network: options.home_network, listen_addr: None, local: options.local, log_dir_path: service_log_dir_path.clone(), diff --git a/sn_node_manager/src/add_services/tests.rs b/sn_node_manager/src/add_services/tests.rs index 3e3ac0669a..20ccc28717 100644 --- a/sn_node_manager/src/add_services/tests.rs +++ b/sn_node_manager/src/add_services/tests.rs @@ -109,6 +109,7 @@ async fn add_genesis_node_should_use_latest_version_and_add_one_service() -> Res data_dir_path: node_data_dir.to_path_buf().join("safenode1"), env_variables: None, genesis: true, + home_network: false, local: true, log_dir_path: node_logs_dir.to_path_buf().join("safenode1"), metrics_port: None, @@ -136,6 +137,7 @@ async fn add_genesis_node_should_use_latest_version_and_add_one_service() -> Res delete_safenode_src: true, env_variables: None, genesis: true, + home_network: false, local: true, metrics_port: None, node_port: None, @@ -195,21 +197,22 @@ async fn add_genesis_node_should_return_an_error_if_there_is_already_a_genesis_n faucet: None, save_path: node_reg_path.to_path_buf(), nodes: vec![NodeServiceData { + connected_peers: None, + data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: true, + home_network: false, + listen_addr: None, local: false, - service_name: "safenode1".to_string(), - user: "safe".to_string(), + log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), number: 1, - rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), - version: latest_version.to_string(), - status: ServiceStatus::Added, - listen_addr: None, pid: None, peer_id: None, - log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), - data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), + rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), + status: ServiceStatus::Added, safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), - connected_peers: None, + service_name: "safenode1".to_string(), + user: "safe".to_string(), + version: latest_version.to_string(), }], bootstrap_peers: vec![], environment_variables: None, @@ -233,6 +236,7 @@ async fn add_genesis_node_should_return_an_error_if_there_is_already_a_genesis_n delete_safenode_src: true, env_variables: None, genesis: true, + home_network: false, local: true, metrics_port: None, node_port: None, @@ -291,6 +295,7 @@ async fn add_genesis_node_should_return_an_error_if_count_is_greater_than_1() -> delete_safenode_src: true, env_variables: None, genesis: true, + home_network: false, local: true, metrics_port: None, node_port: None, @@ -356,6 +361,7 @@ async fn add_node_should_use_latest_version_and_add_three_services() -> Result<( data_dir_path: node_data_dir.to_path_buf().join("safenode1"), env_variables: None, genesis: false, + home_network: false, local: false, log_dir_path: node_logs_dir.to_path_buf().join("safenode1"), metrics_port: None, @@ -388,6 +394,7 @@ async fn add_node_should_use_latest_version_and_add_three_services() -> Result<( data_dir_path: node_data_dir.to_path_buf().join("safenode2"), env_variables: None, genesis: false, + home_network: false, local: false, log_dir_path: node_logs_dir.to_path_buf().join("safenode2"), metrics_port: None, @@ -420,6 +427,7 @@ async fn add_node_should_use_latest_version_and_add_three_services() -> Result<( bootstrap_peers: vec![], env_variables: None, genesis: false, + home_network: false, local: false, log_dir_path: node_logs_dir.to_path_buf().join("safenode3"), metrics_port: None, @@ -448,6 +456,7 @@ async fn add_node_should_use_latest_version_and_add_three_services() -> Result<( delete_safenode_src: true, env_variables: None, genesis: false, + home_network: false, local: false, metrics_port: None, node_port: None, @@ -562,6 +571,7 @@ async fn add_node_should_update_the_bootstrap_peers_inside_node_registry() -> Re data_dir_path: node_data_dir.to_path_buf().join("safenode1"), env_variables: None, genesis: false, + home_network: false, local: false, log_dir_path: node_logs_dir.to_path_buf().join("safenode1"), metrics_port: None, @@ -590,6 +600,7 @@ async fn add_node_should_update_the_bootstrap_peers_inside_node_registry() -> Re env_variables: None, local: false, genesis: false, + home_network: false, metrics_port: None, node_port: None, rpc_address: None, @@ -677,6 +688,7 @@ async fn add_node_should_update_the_environment_variables_inside_node_registry() data_dir_path: node_data_dir.to_path_buf().join("safenode1"), env_variables: env_variables.clone(), genesis: false, + home_network: false, local: false, log_dir_path: node_logs_dir.to_path_buf().join("safenode1"), metrics_port: None, @@ -704,6 +716,7 @@ async fn add_node_should_update_the_environment_variables_inside_node_registry() delete_safenode_src: true, env_variables: env_variables.clone(), genesis: false, + home_network: false, local: false, metrics_port: None, node_port: None, @@ -762,21 +775,22 @@ async fn add_new_node_should_add_another_service() -> Result<()> { faucet: None, save_path: node_reg_path.to_path_buf(), nodes: vec![NodeServiceData { + connected_peers: None, + data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: true, + home_network: false, + listen_addr: None, local: false, - service_name: "safenode1".to_string(), - user: "safe".to_string(), + log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), number: 1, - rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), - version: latest_version.to_string(), - status: ServiceStatus::Added, pid: None, peer_id: None, - listen_addr: None, - log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), - data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), + rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), - connected_peers: None, + service_name: "safenode1".to_string(), + status: ServiceStatus::Added, + user: "safe".to_string(), + version: latest_version.to_string(), }], bootstrap_peers: vec![], environment_variables: None, @@ -801,6 +815,7 @@ async fn add_new_node_should_add_another_service() -> Result<()> { data_dir_path: node_data_dir.to_path_buf().join("safenode2"), env_variables: None, genesis: false, + home_network: false, local: false, log_dir_path: node_logs_dir.to_path_buf().join("safenode2"), metrics_port: None, @@ -829,6 +844,7 @@ async fn add_new_node_should_add_another_service() -> Result<()> { delete_safenode_src: true, env_variables: None, genesis: false, + home_network: false, local: false, metrics_port: None, node_port: None, @@ -907,6 +923,7 @@ async fn add_node_should_use_custom_ports_for_one_service() -> Result<()> { data_dir_path: node_data_dir.to_path_buf().join("safenode1"), env_variables: None, genesis: false, + home_network: false, local: false, log_dir_path: node_logs_dir.to_path_buf().join("safenode1"), metrics_port: None, @@ -935,6 +952,7 @@ async fn add_node_should_use_custom_ports_for_one_service() -> Result<()> { delete_safenode_src: true, env_variables: None, genesis: false, + home_network: false, local: false, metrics_port: None, node_port: Some(PortRange::Single(custom_port)), @@ -1147,6 +1165,7 @@ async fn add_node_should_use_a_custom_port_range() -> Result<()> { delete_safenode_src: true, env_variables: None, genesis: false, + home_network: false, local: false, metrics_port: None, node_port: Some(PortRange::Range(12000, 12002)), @@ -1202,6 +1221,7 @@ async fn add_node_should_return_an_error_if_port_and_node_count_do_not_match() - delete_safenode_src: true, env_variables: None, genesis: false, + home_network: false, local: false, metrics_port: None, node_port: Some(PortRange::Range(12000, 12002)), @@ -1263,6 +1283,7 @@ async fn add_node_should_return_an_error_if_multiple_services_are_specified_with delete_safenode_src: true, env_variables: None, genesis: false, + home_network: false, local: false, metrics_port: None, node_port: Some(PortRange::Single(12000)), @@ -1462,6 +1483,7 @@ async fn add_node_should_use_a_custom_port_range_for_metrics_server() -> Result< delete_safenode_src: true, env_variables: None, genesis: false, + home_network: false, local: false, metrics_port: Some(PortRange::Range(12000, 12002)), node_port: None, @@ -1635,6 +1657,7 @@ async fn add_node_should_use_a_custom_port_range_for_the_rpc_server() -> Result< delete_safenode_src: true, env_variables: None, genesis: false, + home_network: false, local: false, metrics_port: None, node_port: None, @@ -1989,6 +2012,7 @@ async fn add_node_should_not_delete_the_source_binary_if_path_arg_is_used() -> R data_dir_path: node_data_dir.to_path_buf().join("safenode1"), env_variables: None, genesis: false, + home_network: false, local: false, log_dir_path: node_logs_dir.to_path_buf().join("safenode1"), metrics_port: None, @@ -2017,6 +2041,7 @@ async fn add_node_should_not_delete_the_source_binary_if_path_arg_is_used() -> R delete_safenode_src: false, env_variables: None, genesis: false, + home_network: false, local: false, metrics_port: None, node_port: None, @@ -2039,3 +2064,95 @@ async fn add_node_should_not_delete_the_source_binary_if_path_arg_is_used() -> R Ok(()) } + +#[tokio::test] +async fn add_node_should_apply_the_home_network_flag_if_it_is_used() -> Result<()> { + let tmp_data_dir = assert_fs::TempDir::new()?; + let node_reg_path = tmp_data_dir.child("node_reg.json"); + + let mut mock_service_control = MockServiceControl::new(); + + let mut node_registry = NodeRegistry { + faucet: None, + save_path: node_reg_path.to_path_buf(), + nodes: vec![], + bootstrap_peers: vec![], + environment_variables: None, + daemon: None, + }; + + let latest_version = "0.96.4"; + let temp_dir = assert_fs::TempDir::new()?; + let node_data_dir = temp_dir.child("data"); + node_data_dir.create_dir_all()?; + let node_logs_dir = temp_dir.child("logs"); + node_logs_dir.create_dir_all()?; + let safenode_download_path = temp_dir.child(SAFENODE_FILE_NAME); + safenode_download_path.write_binary(b"fake safenode bin")?; + + let mut seq = Sequence::new(); + + // Expected calls for first installation + mock_service_control + .expect_get_available_port() + .times(1) + .returning(|| Ok(8081)) + .in_sequence(&mut seq); + + let install_ctx = InstallNodeServiceCtxBuilder { + bootstrap_peers: vec![], + data_dir_path: node_data_dir.to_path_buf().join("safenode1"), + env_variables: None, + genesis: false, + home_network: true, + local: false, + log_dir_path: node_logs_dir.to_path_buf().join("safenode1"), + metrics_port: None, + name: "safenode1".to_string(), + node_port: None, + rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), + safenode_path: node_data_dir + .to_path_buf() + .join("safenode1") + .join(SAFENODE_FILE_NAME), + service_user: get_username(), + } + .build()?; + + mock_service_control + .expect_install() + .times(1) + .with(eq(install_ctx)) + .returning(|_| Ok(())) + .in_sequence(&mut seq); + + add_node( + AddNodeServiceOptions { + bootstrap_peers: vec![], + count: Some(1), + delete_safenode_src: false, + env_variables: None, + genesis: false, + home_network: true, + local: false, + metrics_port: None, + node_port: None, + rpc_address: None, + rpc_port: None, + safenode_dir_path: temp_dir.to_path_buf(), + safenode_src_path: safenode_download_path.to_path_buf(), + service_data_dir_path: node_data_dir.to_path_buf(), + service_log_dir_path: node_logs_dir.to_path_buf(), + user: get_username(), + version: latest_version.to_string(), + }, + &mut node_registry, + &mock_service_control, + VerbosityLevel::Normal, + ) + .await?; + + assert!(node_registry.nodes[0].home_network); + + Ok(()) +} diff --git a/sn_node_manager/src/bin/cli/main.rs b/sn_node_manager/src/bin/cli/main.rs index cbcc3939b6..fd5b641bb3 100644 --- a/sn_node_manager/src/bin/cli/main.rs +++ b/sn_node_manager/src/bin/cli/main.rs @@ -61,6 +61,11 @@ pub enum SubCmd { /// Example: --env SN_LOG=all,RUST_LOG=libp2p=debug #[clap(name = "env", long, use_value_delimiter = true, value_parser = parse_environment_variables)] env_variables: Option>, + /// Set this flag to use the safenode '--home-network' feature. + /// + /// This enables the use of safenode services from a home network with a router. + #[clap(long)] + home_network: bool, /// Set this flag to launch safenode with the --local flag. /// /// This is useful for building a service-based local network. @@ -609,6 +614,7 @@ async fn main() -> Result<()> { count, data_dir_path, env_variables, + home_network, local, log_dir_path, metrics_port, @@ -625,6 +631,7 @@ async fn main() -> Result<()> { count, data_dir_path, env_variables, + home_network, local, log_dir_path, metrics_port, diff --git a/sn_node_manager/src/cmd/node.rs b/sn_node_manager/src/cmd/node.rs index e2deac217c..aa563922ae 100644 --- a/sn_node_manager/src/cmd/node.rs +++ b/sn_node_manager/src/cmd/node.rs @@ -38,6 +38,7 @@ pub async fn add( count: Option, data_dir_path: Option, env_variables: Option>, + home_network: bool, local: bool, log_dir_path: Option, metrics_port: Option, @@ -105,6 +106,7 @@ pub async fn add( delete_safenode_src: src_path.is_none(), env_variables, genesis: is_first, + home_network, local, metrics_port, node_port, diff --git a/sn_node_manager/src/lib.rs b/sn_node_manager/src/lib.rs index 25dfe8a231..f1e033efb5 100644 --- a/sn_node_manager/src/lib.rs +++ b/sn_node_manager/src/lib.rs @@ -571,6 +571,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -654,6 +655,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -708,6 +710,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -798,6 +801,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -867,6 +871,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -915,6 +920,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -954,6 +960,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -991,6 +998,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -1031,6 +1039,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -1144,6 +1153,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -1221,6 +1231,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -1340,6 +1351,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -1472,6 +1484,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -1600,6 +1613,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -1665,21 +1679,22 @@ mod tests { .returning(|_| Ok(())); let mut service_data = NodeServiceData { + connected_peers: None, + data_dir_path: data_dir.to_path_buf(), genesis: false, + home_network: false, + listen_addr: None, local: false, - version: "0.98.1".to_string(), - service_name: "safenode1".to_string(), - user: "safe".to_string(), + log_dir_path: log_dir.to_path_buf(), number: 1, - rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), - status: ServiceStatus::Stopped, pid: None, peer_id: None, - listen_addr: None, - log_dir_path: log_dir.to_path_buf(), - data_dir_path: data_dir.to_path_buf(), + rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: safenode_bin.to_path_buf(), - connected_peers: None, + status: ServiceStatus::Stopped, + service_name: "safenode1".to_string(), + version: "0.98.1".to_string(), + user: "safe".to_string(), }; let service = NodeService::new(&mut service_data, Box::new(MockRpcClient::new())); let mut service_manager = ServiceManager::new( @@ -1710,23 +1725,24 @@ mod tests { .returning(|_| true); let mut service_data = NodeServiceData { + connected_peers: None, + data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, + listen_addr: None, local: false, - version: "0.98.1".to_string(), - service_name: "safenode1".to_string(), - user: "safe".to_string(), + log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), number: 1, - rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), - status: ServiceStatus::Running, pid: Some(1000), peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), - listen_addr: None, - log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), - data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), + rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), - connected_peers: None, + service_name: "safenode1".to_string(), + status: ServiceStatus::Running, + user: "safe".to_string(), + version: "0.98.1".to_string(), }; let service = NodeService::new(&mut service_data, Box::new(MockRpcClient::new())); let mut service_manager = ServiceManager::new( @@ -1763,23 +1779,24 @@ mod tests { .returning(|_| false); let mut service_data = NodeServiceData { + connected_peers: None, + data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, + listen_addr: None, local: false, - version: "0.98.1".to_string(), - service_name: "safenode1".to_string(), - user: "safe".to_string(), + log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), number: 1, rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), + safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), + service_name: "safenode1".to_string(), status: ServiceStatus::Running, pid: Some(1000), peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), - listen_addr: None, - log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), - data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), - safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), - connected_peers: None, + user: "safe".to_string(), + version: "0.98.1".to_string(), }; let service = NodeService::new(&mut service_data, Box::new(MockRpcClient::new())); let mut service_manager = ServiceManager::new( @@ -1818,21 +1835,22 @@ mod tests { .returning(|_| Ok(())); let mut service_data = NodeServiceData { + connected_peers: None, + data_dir_path: data_dir.to_path_buf(), genesis: false, + home_network: false, + listen_addr: None, local: false, - version: "0.98.1".to_string(), - service_name: "safenode1".to_string(), - user: "safe".to_string(), + log_dir_path: log_dir.to_path_buf(), number: 1, - rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), - status: ServiceStatus::Stopped, pid: None, peer_id: None, - listen_addr: None, - log_dir_path: log_dir.to_path_buf(), - data_dir_path: data_dir.to_path_buf(), + rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: safenode_bin.to_path_buf(), - connected_peers: None, + service_name: "safenode1".to_string(), + status: ServiceStatus::Stopped, + user: "safe".to_string(), + version: "0.98.1".to_string(), }; let service = NodeService::new(&mut service_data, Box::new(MockRpcClient::new())); let mut service_manager = ServiceManager::new( diff --git a/sn_node_manager/src/local.rs b/sn_node_manager/src/local.rs index 6f2a9d3dff..3d270c84e2 100644 --- a/sn_node_manager/src/local.rs +++ b/sn_node_manager/src/local.rs @@ -306,7 +306,7 @@ pub async fn run_node( Ok(NodeServiceData { connected_peers, genesis: run_options.genesis, - // not read for local network. + home_network: false, local: true, service_name: format!("safenode-local{}", run_options.number), user: get_username()?, diff --git a/sn_node_manager/src/rpc.rs b/sn_node_manager/src/rpc.rs index 14884ec522..fa558ece66 100644 --- a/sn_node_manager/src/rpc.rs +++ b/sn_node_manager/src/rpc.rs @@ -59,6 +59,7 @@ pub async fn restart_node_service( data_dir_path: current_node_clone.data_dir_path.clone(), env_variables: node_registry.environment_variables.clone(), genesis: current_node_clone.genesis, + home_network: current_node_clone.home_network, local: current_node_clone.local, log_dir_path: current_node_clone.log_dir_path.clone(), metrics_port: None, @@ -136,20 +137,19 @@ pub async fn restart_node_service( }; let install_ctx = InstallNodeServiceCtxBuilder { - local: current_node_clone.local, + bootstrap_peers: node_registry.bootstrap_peers.clone(), + data_dir_path: data_dir_path.clone(), + env_variables: node_registry.environment_variables.clone(), genesis: current_node_clone.genesis, + home_network: current_node_clone.home_network, + local: current_node_clone.local, + log_dir_path: log_dir_path.clone(), name: new_service_name.clone(), - // don't re-use port metrics_port: None, node_port: None, - bootstrap_peers: node_registry.bootstrap_peers.clone(), rpc_socket_addr: current_node_clone.rpc_socket_addr, - // set new paths - data_dir_path: data_dir_path.clone(), - log_dir_path: log_dir_path.clone(), safenode_path: safenode_path.clone(), service_user: current_node_clone.user.clone(), - env_variables: node_registry.environment_variables.clone(), } .build()?; service_control.install(install_ctx).map_err(|err| { @@ -157,21 +157,22 @@ pub async fn restart_node_service( })?; let mut node = NodeServiceData { + connected_peers: None, + data_dir_path, genesis: current_node_clone.genesis, + home_network: current_node_clone.home_network, + listen_addr: None, local: current_node_clone.local, - service_name: new_service_name.clone(), - user: current_node_clone.user.clone(), + log_dir_path, number: new_node_number as u16, - rpc_socket_addr: current_node_clone.rpc_socket_addr, - version: current_node_clone.version.clone(), - status: ServiceStatus::Added, - listen_addr: None, - pid: None, peer_id: None, - log_dir_path, - data_dir_path, + pid: None, + rpc_socket_addr: current_node_clone.rpc_socket_addr, safenode_path, - connected_peers: None, + service_name: new_service_name.clone(), + status: ServiceStatus::Added, + user: current_node_clone.user.clone(), + version: current_node_clone.version.clone(), }; let rpc_client = RpcClient::from_socket_addr(node.rpc_socket_addr); diff --git a/sn_service_management/src/node.rs b/sn_service_management/src/node.rs index 77a9a61d32..6fdadb68e7 100644 --- a/sn_service_management/src/node.rs +++ b/sn_service_management/src/node.rs @@ -151,6 +151,7 @@ pub struct NodeServiceData { pub connected_peers: Option>, pub data_dir_path: PathBuf, pub genesis: bool, + pub home_network: bool, pub listen_addr: Option>, pub local: bool, pub log_dir_path: PathBuf, From 9d03d8b40d9fdb3f85b10da03f716aeb70d90e0e Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Thu, 25 Apr 2024 17:22:57 +0530 Subject: [PATCH 142/205] chore(network): remove nat detection via incoming connections check --- sn_networking/src/event.rs | 16 +++++++--------- sn_networking/src/relay_manager.rs | 25 ------------------------- 2 files changed, 7 insertions(+), 34 deletions(-) diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index b912936653..31e04f19c9 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -163,7 +163,6 @@ pub enum NetworkEvent { #[derive(Debug, Clone)] pub enum TerminateNodeReason { HardDiskWriteError, - BehindNAT, } // Manually implement Debug as `#[debug(with = "unverified_record_fmt")]` not working as expected. @@ -257,6 +256,13 @@ impl SwarmDriver { event_string = "kad_event"; self.handle_kad_event(kad_event)?; } + SwarmEvent::Behaviour(NodeEvent::Dcutr(event)) => { + event_string = "dcutr_event"; + info!( + "Dcutr with remote peer: {:?} is: {:?}", + event.remote_peer_id, event.result + ); + } SwarmEvent::Behaviour(NodeEvent::RelayClient(event)) => { event_string = "relay_client_event"; @@ -419,14 +425,6 @@ impl SwarmDriver { .kademlia .add_address(&peer_id, multiaddr.clone()); } - - if self.relay_manager.are_we_behind_nat(&mut self.swarm) { - warn!("Node reported as being behind NAT for not having enough incoming connections. This can be a false positive. Do nothing."); - - // self.send_event(NetworkEvent::TerminateNode { - // reason: TerminateNodeReason::BehindNAT, - // }) - } } } trace!( diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index 58bfcad00f..720a1fc1cf 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -14,7 +14,6 @@ use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; const MAX_CONCURRENT_RELAY_CONNECTIONS: usize = 3; const MAX_POTENTIAL_CANDIDATES: usize = 15; -const MAX_PEERS_IN_RT_DURING_NAT_CHECK: usize = 30; /// To manager relayed connections. #[derive(Debug)] @@ -65,30 +64,6 @@ impl RelayManager { || self.waiting_for_reservation.contains_key(peer_id) } - /// If we have 0 incoming connection even after we have a lot of peers, then we are behind a NAT - pub(crate) fn are_we_behind_nat(&self, swarm: &mut Swarm) -> bool { - if swarm - .network_info() - .connection_counters() - .num_established_incoming() - == 0 - || swarm - .network_info() - .connection_counters() - .num_pending_incoming() - == 0 - { - let mut total_peers = 0; - for kbucket in swarm.behaviour_mut().kademlia.kbuckets() { - total_peers += kbucket.num_entries(); - if total_peers > MAX_PEERS_IN_RT_DURING_NAT_CHECK { - return true; - } - } - } - false - } - /// Add a potential candidate to the list if it satisfies all the identify checks and also supports the relay server /// protocol. pub(crate) fn add_potential_candidates( From 7394858843b36b499d0a8a7748276a5d9b3ace5a Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 24 Apr 2024 18:45:38 +0100 Subject: [PATCH 143/205] fix: do not delete custom bin on `add` cmd When the `--path` argument is specified on the node manager's `add` command, a custom binary is supplied, and now, the source of `--path` is not deleted. By default, the `add` command downloads the latest version of a binary to a temporary location, and that binary is then copied to the service location. Having finished with it, the node manager then cleans up the temporary binary. When the `--path` argument is used, it goes through the same code path, and hence the source of `--path` was being deleted as a side effect. Now, a flag is used to indicate whether the binary should be deleted, and it is set to `false` when the `--path` argument is supplied. The `--path` argument is being used when users build their own `safenode`, so they are expecting that the built binary will still exist. --- sn_node_manager/src/add_services/config.rs | 1 + sn_node_manager/src/add_services/mod.rs | 4 +- sn_node_manager/src/add_services/tests.rs | 103 +++++++++++++++++++++ sn_node_manager/src/cmd/node.rs | 3 +- 4 files changed, 109 insertions(+), 2 deletions(-) diff --git a/sn_node_manager/src/add_services/config.rs b/sn_node_manager/src/add_services/config.rs index ed1855f2a9..70ab0dd713 100644 --- a/sn_node_manager/src/add_services/config.rs +++ b/sn_node_manager/src/add_services/config.rs @@ -108,6 +108,7 @@ impl InstallNodeServiceCtxBuilder { pub struct AddNodeServiceOptions { pub bootstrap_peers: Vec, pub count: Option, + pub delete_safenode_src: bool, pub env_variables: Option>, pub genesis: bool, pub local: bool, diff --git a/sn_node_manager/src/add_services/mod.rs b/sn_node_manager/src/add_services/mod.rs index 37f265e311..2e0232311d 100644 --- a/sn_node_manager/src/add_services/mod.rs +++ b/sn_node_manager/src/add_services/mod.rs @@ -198,7 +198,9 @@ pub async fn add_node( rpc_port = increment_port_option(rpc_port); } - std::fs::remove_file(options.safenode_src_path)?; + if options.delete_safenode_src { + std::fs::remove_file(options.safenode_src_path)?; + } if !added_service_data.is_empty() { println!("Services Added:"); diff --git a/sn_node_manager/src/add_services/tests.rs b/sn_node_manager/src/add_services/tests.rs index b8497b68d2..3e3ac0669a 100644 --- a/sn_node_manager/src/add_services/tests.rs +++ b/sn_node_manager/src/add_services/tests.rs @@ -133,6 +133,7 @@ async fn add_genesis_node_should_use_latest_version_and_add_one_service() -> Res AddNodeServiceOptions { bootstrap_peers: vec![], count: None, + delete_safenode_src: true, env_variables: None, genesis: true, local: true, @@ -229,6 +230,7 @@ async fn add_genesis_node_should_return_an_error_if_there_is_already_a_genesis_n AddNodeServiceOptions { bootstrap_peers: vec![], count: None, + delete_safenode_src: true, env_variables: None, genesis: true, local: true, @@ -286,6 +288,7 @@ async fn add_genesis_node_should_return_an_error_if_count_is_greater_than_1() -> AddNodeServiceOptions { bootstrap_peers: vec![], count: Some(3), + delete_safenode_src: true, env_variables: None, genesis: true, local: true, @@ -442,6 +445,7 @@ async fn add_node_should_use_latest_version_and_add_three_services() -> Result<( AddNodeServiceOptions { bootstrap_peers: vec![], count: Some(3), + delete_safenode_src: true, env_variables: None, genesis: false, local: false, @@ -582,6 +586,7 @@ async fn add_node_should_update_the_bootstrap_peers_inside_node_registry() -> Re AddNodeServiceOptions { bootstrap_peers: new_peers.clone(), count: None, + delete_safenode_src: true, env_variables: None, local: false, genesis: false, @@ -696,6 +701,7 @@ async fn add_node_should_update_the_environment_variables_inside_node_registry() AddNodeServiceOptions { bootstrap_peers: vec![], count: None, + delete_safenode_src: true, env_variables: env_variables.clone(), genesis: false, local: false, @@ -820,6 +826,7 @@ async fn add_new_node_should_add_another_service() -> Result<()> { AddNodeServiceOptions { bootstrap_peers: vec![], count: None, + delete_safenode_src: true, env_variables: None, genesis: false, local: false, @@ -925,6 +932,7 @@ async fn add_node_should_use_custom_ports_for_one_service() -> Result<()> { AddNodeServiceOptions { bootstrap_peers: vec![], count: None, + delete_safenode_src: true, env_variables: None, genesis: false, local: false, @@ -1136,6 +1144,7 @@ async fn add_node_should_use_a_custom_port_range() -> Result<()> { AddNodeServiceOptions { bootstrap_peers: vec![], count: Some(3), + delete_safenode_src: true, env_variables: None, genesis: false, local: false, @@ -1190,6 +1199,7 @@ async fn add_node_should_return_an_error_if_port_and_node_count_do_not_match() - AddNodeServiceOptions { bootstrap_peers: vec![], count: Some(2), + delete_safenode_src: true, env_variables: None, genesis: false, local: false, @@ -1250,6 +1260,7 @@ async fn add_node_should_return_an_error_if_multiple_services_are_specified_with AddNodeServiceOptions { bootstrap_peers: vec![], count: Some(2), + delete_safenode_src: true, env_variables: None, genesis: false, local: false, @@ -1448,6 +1459,7 @@ async fn add_node_should_use_a_custom_port_range_for_metrics_server() -> Result< AddNodeServiceOptions { bootstrap_peers: vec![], count: Some(3), + delete_safenode_src: true, env_variables: None, genesis: false, local: false, @@ -1620,6 +1632,7 @@ async fn add_node_should_use_a_custom_port_range_for_the_rpc_server() -> Result< AddNodeServiceOptions { bootstrap_peers: vec![], count: Some(3), + delete_safenode_src: true, env_variables: None, genesis: false, local: false, @@ -1936,3 +1949,93 @@ async fn add_daemon_should_return_an_error_if_a_daemon_service_was_already_creat Ok(()) } + +#[tokio::test] +async fn add_node_should_not_delete_the_source_binary_if_path_arg_is_used() -> Result<()> { + let tmp_data_dir = assert_fs::TempDir::new()?; + let node_reg_path = tmp_data_dir.child("node_reg.json"); + + let mut mock_service_control = MockServiceControl::new(); + + let mut node_registry = NodeRegistry { + faucet: None, + save_path: node_reg_path.to_path_buf(), + nodes: vec![], + bootstrap_peers: vec![], + environment_variables: None, + daemon: None, + }; + + let latest_version = "0.96.4"; + let temp_dir = assert_fs::TempDir::new()?; + let node_data_dir = temp_dir.child("data"); + node_data_dir.create_dir_all()?; + let node_logs_dir = temp_dir.child("logs"); + node_logs_dir.create_dir_all()?; + let safenode_download_path = temp_dir.child(SAFENODE_FILE_NAME); + safenode_download_path.write_binary(b"fake safenode bin")?; + + let mut seq = Sequence::new(); + + // Expected calls for first installation + mock_service_control + .expect_get_available_port() + .times(1) + .returning(|| Ok(8081)) + .in_sequence(&mut seq); + + let install_ctx = InstallNodeServiceCtxBuilder { + bootstrap_peers: vec![], + data_dir_path: node_data_dir.to_path_buf().join("safenode1"), + env_variables: None, + genesis: false, + local: false, + log_dir_path: node_logs_dir.to_path_buf().join("safenode1"), + metrics_port: None, + name: "safenode1".to_string(), + node_port: None, + rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), + safenode_path: node_data_dir + .to_path_buf() + .join("safenode1") + .join(SAFENODE_FILE_NAME), + service_user: get_username(), + } + .build()?; + + mock_service_control + .expect_install() + .times(1) + .with(eq(install_ctx)) + .returning(|_| Ok(())) + .in_sequence(&mut seq); + + add_node( + AddNodeServiceOptions { + bootstrap_peers: vec![], + count: Some(1), + delete_safenode_src: false, + env_variables: None, + genesis: false, + local: false, + metrics_port: None, + node_port: None, + rpc_address: None, + rpc_port: None, + safenode_dir_path: temp_dir.to_path_buf(), + safenode_src_path: safenode_download_path.to_path_buf(), + service_data_dir_path: node_data_dir.to_path_buf(), + service_log_dir_path: node_logs_dir.to_path_buf(), + user: get_username(), + version: latest_version.to_string(), + }, + &mut node_registry, + &mock_service_control, + VerbosityLevel::Normal, + ) + .await?; + + safenode_download_path.assert(predicate::path::is_file()); + + Ok(()) +} diff --git a/sn_node_manager/src/cmd/node.rs b/sn_node_manager/src/cmd/node.rs index 52091890f1..e2deac217c 100644 --- a/sn_node_manager/src/cmd/node.rs +++ b/sn_node_manager/src/cmd/node.rs @@ -73,7 +73,7 @@ pub async fn add( let mut node_registry = NodeRegistry::load(&config::get_node_registry_path()?)?; let release_repo = ::default_config(); - let (safenode_src_path, version) = if let Some(path) = src_path { + let (safenode_src_path, version) = if let Some(path) = src_path.clone() { let version = get_bin_version(&path)?; (path, version) } else { @@ -102,6 +102,7 @@ pub async fn add( let options = AddNodeServiceOptions { count, + delete_safenode_src: src_path.is_none(), env_variables, genesis: is_first, local, From 321f9b82ea1d2487d260c6b2650ef307a52abab7 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Thu, 25 Apr 2024 15:41:19 +0100 Subject: [PATCH 144/205] feat: provide `--home-network` arg for `add` cmd The `add` command now enables the user to pass the `--home-network` argument to the `safenode` service. BREAKING CHANGE: the addition of the `home_network` field to `NodeServiceData` will mean this version of the node manager will not be able to deserialize old node registry files. --- sn_node_manager/src/add_services/config.rs | 5 + sn_node_manager/src/add_services/mod.rs | 2 + sn_node_manager/src/add_services/tests.rs | 153 ++++++++++++++++++--- sn_node_manager/src/bin/cli/main.rs | 7 + sn_node_manager/src/cmd/node.rs | 2 + sn_node_manager/src/lib.rs | 88 +++++++----- sn_node_manager/src/local.rs | 2 +- sn_node_manager/src/rpc.rs | 35 ++--- sn_service_management/src/node.rs | 1 + 9 files changed, 224 insertions(+), 71 deletions(-) diff --git a/sn_node_manager/src/add_services/config.rs b/sn_node_manager/src/add_services/config.rs index 70ab0dd713..fbfa9a696c 100644 --- a/sn_node_manager/src/add_services/config.rs +++ b/sn_node_manager/src/add_services/config.rs @@ -45,6 +45,7 @@ pub struct InstallNodeServiceCtxBuilder { pub data_dir_path: PathBuf, pub env_variables: Option>, pub genesis: bool, + pub home_network: bool, pub local: bool, pub log_dir_path: PathBuf, pub name: String, @@ -70,6 +71,9 @@ impl InstallNodeServiceCtxBuilder { if self.genesis { args.push(OsString::from("--first")); } + if self.home_network { + args.push(OsString::from("--home-network")); + } if self.local { args.push(OsString::from("--local")); } @@ -111,6 +115,7 @@ pub struct AddNodeServiceOptions { pub delete_safenode_src: bool, pub env_variables: Option>, pub genesis: bool, + pub home_network: bool, pub local: bool, pub metrics_port: Option, pub node_port: Option, diff --git a/sn_node_manager/src/add_services/mod.rs b/sn_node_manager/src/add_services/mod.rs index 2e0232311d..9192124eb8 100644 --- a/sn_node_manager/src/add_services/mod.rs +++ b/sn_node_manager/src/add_services/mod.rs @@ -145,6 +145,7 @@ pub async fn add_node( data_dir_path: service_data_dir_path.clone(), env_variables: options.env_variables.clone(), genesis: options.genesis, + home_network: options.home_network, local: options.local, log_dir_path: service_log_dir_path.clone(), metrics_port, @@ -170,6 +171,7 @@ pub async fn add_node( connected_peers: None, data_dir_path: service_data_dir_path.clone(), genesis: options.genesis, + home_network: options.home_network, listen_addr: None, local: options.local, log_dir_path: service_log_dir_path.clone(), diff --git a/sn_node_manager/src/add_services/tests.rs b/sn_node_manager/src/add_services/tests.rs index 3e3ac0669a..20ccc28717 100644 --- a/sn_node_manager/src/add_services/tests.rs +++ b/sn_node_manager/src/add_services/tests.rs @@ -109,6 +109,7 @@ async fn add_genesis_node_should_use_latest_version_and_add_one_service() -> Res data_dir_path: node_data_dir.to_path_buf().join("safenode1"), env_variables: None, genesis: true, + home_network: false, local: true, log_dir_path: node_logs_dir.to_path_buf().join("safenode1"), metrics_port: None, @@ -136,6 +137,7 @@ async fn add_genesis_node_should_use_latest_version_and_add_one_service() -> Res delete_safenode_src: true, env_variables: None, genesis: true, + home_network: false, local: true, metrics_port: None, node_port: None, @@ -195,21 +197,22 @@ async fn add_genesis_node_should_return_an_error_if_there_is_already_a_genesis_n faucet: None, save_path: node_reg_path.to_path_buf(), nodes: vec![NodeServiceData { + connected_peers: None, + data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: true, + home_network: false, + listen_addr: None, local: false, - service_name: "safenode1".to_string(), - user: "safe".to_string(), + log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), number: 1, - rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), - version: latest_version.to_string(), - status: ServiceStatus::Added, - listen_addr: None, pid: None, peer_id: None, - log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), - data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), + rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), + status: ServiceStatus::Added, safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), - connected_peers: None, + service_name: "safenode1".to_string(), + user: "safe".to_string(), + version: latest_version.to_string(), }], bootstrap_peers: vec![], environment_variables: None, @@ -233,6 +236,7 @@ async fn add_genesis_node_should_return_an_error_if_there_is_already_a_genesis_n delete_safenode_src: true, env_variables: None, genesis: true, + home_network: false, local: true, metrics_port: None, node_port: None, @@ -291,6 +295,7 @@ async fn add_genesis_node_should_return_an_error_if_count_is_greater_than_1() -> delete_safenode_src: true, env_variables: None, genesis: true, + home_network: false, local: true, metrics_port: None, node_port: None, @@ -356,6 +361,7 @@ async fn add_node_should_use_latest_version_and_add_three_services() -> Result<( data_dir_path: node_data_dir.to_path_buf().join("safenode1"), env_variables: None, genesis: false, + home_network: false, local: false, log_dir_path: node_logs_dir.to_path_buf().join("safenode1"), metrics_port: None, @@ -388,6 +394,7 @@ async fn add_node_should_use_latest_version_and_add_three_services() -> Result<( data_dir_path: node_data_dir.to_path_buf().join("safenode2"), env_variables: None, genesis: false, + home_network: false, local: false, log_dir_path: node_logs_dir.to_path_buf().join("safenode2"), metrics_port: None, @@ -420,6 +427,7 @@ async fn add_node_should_use_latest_version_and_add_three_services() -> Result<( bootstrap_peers: vec![], env_variables: None, genesis: false, + home_network: false, local: false, log_dir_path: node_logs_dir.to_path_buf().join("safenode3"), metrics_port: None, @@ -448,6 +456,7 @@ async fn add_node_should_use_latest_version_and_add_three_services() -> Result<( delete_safenode_src: true, env_variables: None, genesis: false, + home_network: false, local: false, metrics_port: None, node_port: None, @@ -562,6 +571,7 @@ async fn add_node_should_update_the_bootstrap_peers_inside_node_registry() -> Re data_dir_path: node_data_dir.to_path_buf().join("safenode1"), env_variables: None, genesis: false, + home_network: false, local: false, log_dir_path: node_logs_dir.to_path_buf().join("safenode1"), metrics_port: None, @@ -590,6 +600,7 @@ async fn add_node_should_update_the_bootstrap_peers_inside_node_registry() -> Re env_variables: None, local: false, genesis: false, + home_network: false, metrics_port: None, node_port: None, rpc_address: None, @@ -677,6 +688,7 @@ async fn add_node_should_update_the_environment_variables_inside_node_registry() data_dir_path: node_data_dir.to_path_buf().join("safenode1"), env_variables: env_variables.clone(), genesis: false, + home_network: false, local: false, log_dir_path: node_logs_dir.to_path_buf().join("safenode1"), metrics_port: None, @@ -704,6 +716,7 @@ async fn add_node_should_update_the_environment_variables_inside_node_registry() delete_safenode_src: true, env_variables: env_variables.clone(), genesis: false, + home_network: false, local: false, metrics_port: None, node_port: None, @@ -762,21 +775,22 @@ async fn add_new_node_should_add_another_service() -> Result<()> { faucet: None, save_path: node_reg_path.to_path_buf(), nodes: vec![NodeServiceData { + connected_peers: None, + data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: true, + home_network: false, + listen_addr: None, local: false, - service_name: "safenode1".to_string(), - user: "safe".to_string(), + log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), number: 1, - rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), - version: latest_version.to_string(), - status: ServiceStatus::Added, pid: None, peer_id: None, - listen_addr: None, - log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), - data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), + rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), - connected_peers: None, + service_name: "safenode1".to_string(), + status: ServiceStatus::Added, + user: "safe".to_string(), + version: latest_version.to_string(), }], bootstrap_peers: vec![], environment_variables: None, @@ -801,6 +815,7 @@ async fn add_new_node_should_add_another_service() -> Result<()> { data_dir_path: node_data_dir.to_path_buf().join("safenode2"), env_variables: None, genesis: false, + home_network: false, local: false, log_dir_path: node_logs_dir.to_path_buf().join("safenode2"), metrics_port: None, @@ -829,6 +844,7 @@ async fn add_new_node_should_add_another_service() -> Result<()> { delete_safenode_src: true, env_variables: None, genesis: false, + home_network: false, local: false, metrics_port: None, node_port: None, @@ -907,6 +923,7 @@ async fn add_node_should_use_custom_ports_for_one_service() -> Result<()> { data_dir_path: node_data_dir.to_path_buf().join("safenode1"), env_variables: None, genesis: false, + home_network: false, local: false, log_dir_path: node_logs_dir.to_path_buf().join("safenode1"), metrics_port: None, @@ -935,6 +952,7 @@ async fn add_node_should_use_custom_ports_for_one_service() -> Result<()> { delete_safenode_src: true, env_variables: None, genesis: false, + home_network: false, local: false, metrics_port: None, node_port: Some(PortRange::Single(custom_port)), @@ -1147,6 +1165,7 @@ async fn add_node_should_use_a_custom_port_range() -> Result<()> { delete_safenode_src: true, env_variables: None, genesis: false, + home_network: false, local: false, metrics_port: None, node_port: Some(PortRange::Range(12000, 12002)), @@ -1202,6 +1221,7 @@ async fn add_node_should_return_an_error_if_port_and_node_count_do_not_match() - delete_safenode_src: true, env_variables: None, genesis: false, + home_network: false, local: false, metrics_port: None, node_port: Some(PortRange::Range(12000, 12002)), @@ -1263,6 +1283,7 @@ async fn add_node_should_return_an_error_if_multiple_services_are_specified_with delete_safenode_src: true, env_variables: None, genesis: false, + home_network: false, local: false, metrics_port: None, node_port: Some(PortRange::Single(12000)), @@ -1462,6 +1483,7 @@ async fn add_node_should_use_a_custom_port_range_for_metrics_server() -> Result< delete_safenode_src: true, env_variables: None, genesis: false, + home_network: false, local: false, metrics_port: Some(PortRange::Range(12000, 12002)), node_port: None, @@ -1635,6 +1657,7 @@ async fn add_node_should_use_a_custom_port_range_for_the_rpc_server() -> Result< delete_safenode_src: true, env_variables: None, genesis: false, + home_network: false, local: false, metrics_port: None, node_port: None, @@ -1989,6 +2012,7 @@ async fn add_node_should_not_delete_the_source_binary_if_path_arg_is_used() -> R data_dir_path: node_data_dir.to_path_buf().join("safenode1"), env_variables: None, genesis: false, + home_network: false, local: false, log_dir_path: node_logs_dir.to_path_buf().join("safenode1"), metrics_port: None, @@ -2017,6 +2041,7 @@ async fn add_node_should_not_delete_the_source_binary_if_path_arg_is_used() -> R delete_safenode_src: false, env_variables: None, genesis: false, + home_network: false, local: false, metrics_port: None, node_port: None, @@ -2039,3 +2064,95 @@ async fn add_node_should_not_delete_the_source_binary_if_path_arg_is_used() -> R Ok(()) } + +#[tokio::test] +async fn add_node_should_apply_the_home_network_flag_if_it_is_used() -> Result<()> { + let tmp_data_dir = assert_fs::TempDir::new()?; + let node_reg_path = tmp_data_dir.child("node_reg.json"); + + let mut mock_service_control = MockServiceControl::new(); + + let mut node_registry = NodeRegistry { + faucet: None, + save_path: node_reg_path.to_path_buf(), + nodes: vec![], + bootstrap_peers: vec![], + environment_variables: None, + daemon: None, + }; + + let latest_version = "0.96.4"; + let temp_dir = assert_fs::TempDir::new()?; + let node_data_dir = temp_dir.child("data"); + node_data_dir.create_dir_all()?; + let node_logs_dir = temp_dir.child("logs"); + node_logs_dir.create_dir_all()?; + let safenode_download_path = temp_dir.child(SAFENODE_FILE_NAME); + safenode_download_path.write_binary(b"fake safenode bin")?; + + let mut seq = Sequence::new(); + + // Expected calls for first installation + mock_service_control + .expect_get_available_port() + .times(1) + .returning(|| Ok(8081)) + .in_sequence(&mut seq); + + let install_ctx = InstallNodeServiceCtxBuilder { + bootstrap_peers: vec![], + data_dir_path: node_data_dir.to_path_buf().join("safenode1"), + env_variables: None, + genesis: false, + home_network: true, + local: false, + log_dir_path: node_logs_dir.to_path_buf().join("safenode1"), + metrics_port: None, + name: "safenode1".to_string(), + node_port: None, + rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), + safenode_path: node_data_dir + .to_path_buf() + .join("safenode1") + .join(SAFENODE_FILE_NAME), + service_user: get_username(), + } + .build()?; + + mock_service_control + .expect_install() + .times(1) + .with(eq(install_ctx)) + .returning(|_| Ok(())) + .in_sequence(&mut seq); + + add_node( + AddNodeServiceOptions { + bootstrap_peers: vec![], + count: Some(1), + delete_safenode_src: false, + env_variables: None, + genesis: false, + home_network: true, + local: false, + metrics_port: None, + node_port: None, + rpc_address: None, + rpc_port: None, + safenode_dir_path: temp_dir.to_path_buf(), + safenode_src_path: safenode_download_path.to_path_buf(), + service_data_dir_path: node_data_dir.to_path_buf(), + service_log_dir_path: node_logs_dir.to_path_buf(), + user: get_username(), + version: latest_version.to_string(), + }, + &mut node_registry, + &mock_service_control, + VerbosityLevel::Normal, + ) + .await?; + + assert!(node_registry.nodes[0].home_network); + + Ok(()) +} diff --git a/sn_node_manager/src/bin/cli/main.rs b/sn_node_manager/src/bin/cli/main.rs index cbcc3939b6..fd5b641bb3 100644 --- a/sn_node_manager/src/bin/cli/main.rs +++ b/sn_node_manager/src/bin/cli/main.rs @@ -61,6 +61,11 @@ pub enum SubCmd { /// Example: --env SN_LOG=all,RUST_LOG=libp2p=debug #[clap(name = "env", long, use_value_delimiter = true, value_parser = parse_environment_variables)] env_variables: Option>, + /// Set this flag to use the safenode '--home-network' feature. + /// + /// This enables the use of safenode services from a home network with a router. + #[clap(long)] + home_network: bool, /// Set this flag to launch safenode with the --local flag. /// /// This is useful for building a service-based local network. @@ -609,6 +614,7 @@ async fn main() -> Result<()> { count, data_dir_path, env_variables, + home_network, local, log_dir_path, metrics_port, @@ -625,6 +631,7 @@ async fn main() -> Result<()> { count, data_dir_path, env_variables, + home_network, local, log_dir_path, metrics_port, diff --git a/sn_node_manager/src/cmd/node.rs b/sn_node_manager/src/cmd/node.rs index e2deac217c..aa563922ae 100644 --- a/sn_node_manager/src/cmd/node.rs +++ b/sn_node_manager/src/cmd/node.rs @@ -38,6 +38,7 @@ pub async fn add( count: Option, data_dir_path: Option, env_variables: Option>, + home_network: bool, local: bool, log_dir_path: Option, metrics_port: Option, @@ -105,6 +106,7 @@ pub async fn add( delete_safenode_src: src_path.is_none(), env_variables, genesis: is_first, + home_network, local, metrics_port, node_port, diff --git a/sn_node_manager/src/lib.rs b/sn_node_manager/src/lib.rs index 25dfe8a231..f1e033efb5 100644 --- a/sn_node_manager/src/lib.rs +++ b/sn_node_manager/src/lib.rs @@ -571,6 +571,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -654,6 +655,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -708,6 +710,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -798,6 +801,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -867,6 +871,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -915,6 +920,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -954,6 +960,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -991,6 +998,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -1031,6 +1039,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -1144,6 +1153,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -1221,6 +1231,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -1340,6 +1351,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -1472,6 +1484,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -1600,6 +1613,7 @@ mod tests { connected_peers: None, data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, listen_addr: None, local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), @@ -1665,21 +1679,22 @@ mod tests { .returning(|_| Ok(())); let mut service_data = NodeServiceData { + connected_peers: None, + data_dir_path: data_dir.to_path_buf(), genesis: false, + home_network: false, + listen_addr: None, local: false, - version: "0.98.1".to_string(), - service_name: "safenode1".to_string(), - user: "safe".to_string(), + log_dir_path: log_dir.to_path_buf(), number: 1, - rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), - status: ServiceStatus::Stopped, pid: None, peer_id: None, - listen_addr: None, - log_dir_path: log_dir.to_path_buf(), - data_dir_path: data_dir.to_path_buf(), + rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: safenode_bin.to_path_buf(), - connected_peers: None, + status: ServiceStatus::Stopped, + service_name: "safenode1".to_string(), + version: "0.98.1".to_string(), + user: "safe".to_string(), }; let service = NodeService::new(&mut service_data, Box::new(MockRpcClient::new())); let mut service_manager = ServiceManager::new( @@ -1710,23 +1725,24 @@ mod tests { .returning(|_| true); let mut service_data = NodeServiceData { + connected_peers: None, + data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, + listen_addr: None, local: false, - version: "0.98.1".to_string(), - service_name: "safenode1".to_string(), - user: "safe".to_string(), + log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), number: 1, - rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), - status: ServiceStatus::Running, pid: Some(1000), peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), - listen_addr: None, - log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), - data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), + rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), - connected_peers: None, + service_name: "safenode1".to_string(), + status: ServiceStatus::Running, + user: "safe".to_string(), + version: "0.98.1".to_string(), }; let service = NodeService::new(&mut service_data, Box::new(MockRpcClient::new())); let mut service_manager = ServiceManager::new( @@ -1763,23 +1779,24 @@ mod tests { .returning(|_| false); let mut service_data = NodeServiceData { + connected_peers: None, + data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), genesis: false, + home_network: false, + listen_addr: None, local: false, - version: "0.98.1".to_string(), - service_name: "safenode1".to_string(), - user: "safe".to_string(), + log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), number: 1, rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), + safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), + service_name: "safenode1".to_string(), status: ServiceStatus::Running, pid: Some(1000), peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), - listen_addr: None, - log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), - data_dir_path: PathBuf::from("/var/safenode-manager/services/safenode1"), - safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), - connected_peers: None, + user: "safe".to_string(), + version: "0.98.1".to_string(), }; let service = NodeService::new(&mut service_data, Box::new(MockRpcClient::new())); let mut service_manager = ServiceManager::new( @@ -1818,21 +1835,22 @@ mod tests { .returning(|_| Ok(())); let mut service_data = NodeServiceData { + connected_peers: None, + data_dir_path: data_dir.to_path_buf(), genesis: false, + home_network: false, + listen_addr: None, local: false, - version: "0.98.1".to_string(), - service_name: "safenode1".to_string(), - user: "safe".to_string(), + log_dir_path: log_dir.to_path_buf(), number: 1, - rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), - status: ServiceStatus::Stopped, pid: None, peer_id: None, - listen_addr: None, - log_dir_path: log_dir.to_path_buf(), - data_dir_path: data_dir.to_path_buf(), + rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: safenode_bin.to_path_buf(), - connected_peers: None, + service_name: "safenode1".to_string(), + status: ServiceStatus::Stopped, + user: "safe".to_string(), + version: "0.98.1".to_string(), }; let service = NodeService::new(&mut service_data, Box::new(MockRpcClient::new())); let mut service_manager = ServiceManager::new( diff --git a/sn_node_manager/src/local.rs b/sn_node_manager/src/local.rs index 6f2a9d3dff..3d270c84e2 100644 --- a/sn_node_manager/src/local.rs +++ b/sn_node_manager/src/local.rs @@ -306,7 +306,7 @@ pub async fn run_node( Ok(NodeServiceData { connected_peers, genesis: run_options.genesis, - // not read for local network. + home_network: false, local: true, service_name: format!("safenode-local{}", run_options.number), user: get_username()?, diff --git a/sn_node_manager/src/rpc.rs b/sn_node_manager/src/rpc.rs index 14884ec522..fa558ece66 100644 --- a/sn_node_manager/src/rpc.rs +++ b/sn_node_manager/src/rpc.rs @@ -59,6 +59,7 @@ pub async fn restart_node_service( data_dir_path: current_node_clone.data_dir_path.clone(), env_variables: node_registry.environment_variables.clone(), genesis: current_node_clone.genesis, + home_network: current_node_clone.home_network, local: current_node_clone.local, log_dir_path: current_node_clone.log_dir_path.clone(), metrics_port: None, @@ -136,20 +137,19 @@ pub async fn restart_node_service( }; let install_ctx = InstallNodeServiceCtxBuilder { - local: current_node_clone.local, + bootstrap_peers: node_registry.bootstrap_peers.clone(), + data_dir_path: data_dir_path.clone(), + env_variables: node_registry.environment_variables.clone(), genesis: current_node_clone.genesis, + home_network: current_node_clone.home_network, + local: current_node_clone.local, + log_dir_path: log_dir_path.clone(), name: new_service_name.clone(), - // don't re-use port metrics_port: None, node_port: None, - bootstrap_peers: node_registry.bootstrap_peers.clone(), rpc_socket_addr: current_node_clone.rpc_socket_addr, - // set new paths - data_dir_path: data_dir_path.clone(), - log_dir_path: log_dir_path.clone(), safenode_path: safenode_path.clone(), service_user: current_node_clone.user.clone(), - env_variables: node_registry.environment_variables.clone(), } .build()?; service_control.install(install_ctx).map_err(|err| { @@ -157,21 +157,22 @@ pub async fn restart_node_service( })?; let mut node = NodeServiceData { + connected_peers: None, + data_dir_path, genesis: current_node_clone.genesis, + home_network: current_node_clone.home_network, + listen_addr: None, local: current_node_clone.local, - service_name: new_service_name.clone(), - user: current_node_clone.user.clone(), + log_dir_path, number: new_node_number as u16, - rpc_socket_addr: current_node_clone.rpc_socket_addr, - version: current_node_clone.version.clone(), - status: ServiceStatus::Added, - listen_addr: None, - pid: None, peer_id: None, - log_dir_path, - data_dir_path, + pid: None, + rpc_socket_addr: current_node_clone.rpc_socket_addr, safenode_path, - connected_peers: None, + service_name: new_service_name.clone(), + status: ServiceStatus::Added, + user: current_node_clone.user.clone(), + version: current_node_clone.version.clone(), }; let rpc_client = RpcClient::from_socket_addr(node.rpc_socket_addr); diff --git a/sn_service_management/src/node.rs b/sn_service_management/src/node.rs index 77a9a61d32..6fdadb68e7 100644 --- a/sn_service_management/src/node.rs +++ b/sn_service_management/src/node.rs @@ -151,6 +151,7 @@ pub struct NodeServiceData { pub connected_peers: Option>, pub data_dir_path: PathBuf, pub genesis: bool, + pub home_network: bool, pub listen_addr: Option>, pub local: bool, pub log_dir_path: PathBuf, From 88094893e3dd341ff4e025beac8e6cb62bef96e6 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 25 Apr 2024 16:30:21 +0000 Subject: [PATCH 145/205] chore(release): sn_client-v0.106.0-alpha.5/sn_networking-v0.15.0-alpha.5/sn_cli-v0.91.0-alpha.5/sn_node-v0.106.0-alpha.5/sn_service_management-v0.2.5-alpha.1/sn-node-manager-v0.7.5-alpha.3/sn_auditor-v0.1.12-alpha.0/sn_faucet-v0.4.14-alpha.0/sn_node_rpc_client-v0.6.13-alpha.0 --- Cargo.lock | 18 +++++++++--------- sn_auditor/Cargo.toml | 4 ++-- sn_cli/Cargo.toml | 6 +++--- sn_client/Cargo.toml | 4 ++-- sn_faucet/Cargo.toml | 4 ++-- sn_networking/Cargo.toml | 2 +- sn_node/Cargo.toml | 8 ++++---- sn_node_manager/Cargo.toml | 4 ++-- sn_node_rpc_client/Cargo.toml | 8 ++++---- sn_service_management/Cargo.toml | 2 +- 10 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7548fe3e91..ad5796c663 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5265,7 +5265,7 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "sn-node-manager" -version = "0.7.5-alpha.2" +version = "0.7.5-alpha.3" dependencies = [ "assert_cmd", "assert_fs", @@ -5322,7 +5322,7 @@ dependencies = [ [[package]] name = "sn_auditor" -version = "0.1.11-alpha.0" +version = "0.1.12-alpha.0" dependencies = [ "blsttc", "clap", @@ -5348,7 +5348,7 @@ dependencies = [ [[package]] name = "sn_cli" -version = "0.91.0-alpha.4" +version = "0.91.0-alpha.5" dependencies = [ "aes 0.7.5", "base64 0.22.0", @@ -5389,7 +5389,7 @@ dependencies = [ [[package]] name = "sn_client" -version = "0.106.0-alpha.4" +version = "0.106.0-alpha.5" dependencies = [ "assert_matches", "async-trait", @@ -5438,7 +5438,7 @@ dependencies = [ [[package]] name = "sn_faucet" -version = "0.4.13-alpha.0" +version = "0.4.14-alpha.0" dependencies = [ "assert_fs", "base64 0.22.0", @@ -5505,7 +5505,7 @@ dependencies = [ [[package]] name = "sn_networking" -version = "0.15.0-alpha.4" +version = "0.15.0-alpha.5" dependencies = [ "aes-gcm-siv", "async-trait", @@ -5545,7 +5545,7 @@ dependencies = [ [[package]] name = "sn_node" -version = "0.106.0-alpha.4" +version = "0.106.0-alpha.5" dependencies = [ "assert_fs", "assert_matches", @@ -5600,7 +5600,7 @@ dependencies = [ [[package]] name = "sn_node_rpc_client" -version = "0.6.12-alpha.0" +version = "0.6.13-alpha.0" dependencies = [ "assert_fs", "async-trait", @@ -5689,7 +5689,7 @@ dependencies = [ [[package]] name = "sn_service_management" -version = "0.2.5-alpha.0" +version = "0.2.5-alpha.1" dependencies = [ "async-trait", "dirs-next", diff --git a/sn_auditor/Cargo.toml b/sn_auditor/Cargo.toml index d88446cbeb..37b16d46b2 100644 --- a/sn_auditor/Cargo.toml +++ b/sn_auditor/Cargo.toml @@ -2,7 +2,7 @@ authors = ["MaidSafe Developers "] description = "Safe Network Auditor" name = "sn_auditor" -version = "0.1.11-alpha.0" +version = "0.1.12-alpha.0" edition = "2021" homepage = "https://maidsafe.net" repository = "https://github.com/maidsafe/safe_network" @@ -24,7 +24,7 @@ dirs-next = "~2.0.0" graphviz-rust = "0.9.0" serde = { version = "1.0.133", features = [ "derive", "rc" ]} serde_json = "1.0.108" -sn_client = { path = "../sn_client", version = "0.106.0-alpha.4" } +sn_client = { path = "../sn_client", version = "0.106.0-alpha.5" } sn_logging = { path = "../sn_logging", version = "0.2.26-alpha.0" } sn_peers_acquisition= { path="../sn_peers_acquisition", version = "0.2.11-alpha.0" } tiny_http = { version="0.12", features = ["ssl-rustls"] } diff --git a/sn_cli/Cargo.toml b/sn_cli/Cargo.toml index 13cbeccc12..e9dd5a225f 100644 --- a/sn_cli/Cargo.toml +++ b/sn_cli/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_cli" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.91.0-alpha.4" +version = "0.91.0-alpha.5" [[bin]] path="src/bin/main.rs" @@ -53,7 +53,7 @@ reqwest = { version="0.12.2", default-features=false, features = ["rustls-tls-ma rmp-serde = "1.1.1" serde = { version = "1.0.133", features = [ "derive"]} sn_build_info = { path="../sn_build_info", version = "0.1.7" } -sn_client = { path = "../sn_client", version = "0.106.0-alpha.4" } +sn_client = { path = "../sn_client", version = "0.106.0-alpha.5" } sn_logging = { path = "../sn_logging", version = "0.2.26-alpha.0" } sn_peers_acquisition= { path="../sn_peers_acquisition", version = "0.2.11-alpha.0" } sn_protocol = { path = "../sn_protocol", version = "0.16.4-alpha.0" } @@ -70,7 +70,7 @@ eyre = "0.6.8" criterion = "0.5.1" tempfile = "3.6.0" rand = { version = "~0.8.5", features = ["small_rng"] } -sn_client = { path = "../sn_client", version = "0.106.0-alpha.4", features = ["test-utils"] } +sn_client = { path = "../sn_client", version = "0.106.0-alpha.5", features = ["test-utils"] } [lints] workspace = true diff --git a/sn_client/Cargo.toml b/sn_client/Cargo.toml index 64afbb6bf7..b93c6827e4 100644 --- a/sn_client/Cargo.toml +++ b/sn_client/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_client" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.106.0-alpha.4" +version = "0.106.0-alpha.5" [features] default=[] @@ -36,7 +36,7 @@ rayon = "1.8.0" rmp-serde = "1.1.1" self_encryption = "~0.29.0" serde = { version = "1.0.133", features = [ "derive", "rc" ]} -sn_networking = { path = "../sn_networking", version = "0.15.0-alpha.4" } +sn_networking = { path = "../sn_networking", version = "0.15.0-alpha.5" } sn_protocol = { path = "../sn_protocol", version = "0.16.4-alpha.0" } serde_json = "1.0" sn_registers = { path = "../sn_registers", version = "0.3.13-alpha.0" } diff --git a/sn_faucet/Cargo.toml b/sn_faucet/Cargo.toml index 944731d5d3..1b950f2727 100644 --- a/sn_faucet/Cargo.toml +++ b/sn_faucet/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_faucet" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.4.13-alpha.0" +version = "0.4.14-alpha.0" [features] default = [] @@ -36,7 +36,7 @@ minreq = { version = "2.11.0", features = ["https-rustls"], optional = true } serde = { version = "1.0.193", features = ["derive"] } serde_json = "1.0.108" sn_build_info = { path = "../sn_build_info", version = "0.1.7" } -sn_client = { path = "../sn_client", version = "0.106.0-alpha.4" } +sn_client = { path = "../sn_client", version = "0.106.0-alpha.5" } sn_logging = { path = "../sn_logging", version = "0.2.26-alpha.0" } sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.11-alpha.0" } sn_transfers = { path = "../sn_transfers", version = "0.18.0-alpha.0" } diff --git a/sn_networking/Cargo.toml b/sn_networking/Cargo.toml index 8ce2b5c437..9fd90b6cb6 100644 --- a/sn_networking/Cargo.toml +++ b/sn_networking/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_networking" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.15.0-alpha.4" +version = "0.15.0-alpha.5" [features] default = ["libp2p/quic"] diff --git a/sn_node/Cargo.toml b/sn_node/Cargo.toml index 34038c8347..4f81d8763b 100644 --- a/sn_node/Cargo.toml +++ b/sn_node/Cargo.toml @@ -2,7 +2,7 @@ authors = ["MaidSafe Developers "] description = "Safe Node" name = "sn_node" -version = "0.106.0-alpha.4" +version = "0.106.0-alpha.5" edition = "2021" license = "GPL-3.0" homepage = "https://maidsafe.net" @@ -51,13 +51,13 @@ self_encryption = "~0.29.0" serde = { version = "1.0.133", features = ["derive", "rc"] } sn_build_info = { path = "../sn_build_info", version = "0.1.7" } sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.11-alpha.0" } -sn_client = { path = "../sn_client", version = "0.106.0-alpha.4" } +sn_client = { path = "../sn_client", version = "0.106.0-alpha.5" } sn_logging = { path = "../sn_logging", version = "0.2.26-alpha.0" } -sn_networking = { path = "../sn_networking", version = "0.15.0-alpha.4" } +sn_networking = { path = "../sn_networking", version = "0.15.0-alpha.5" } sn_protocol = { path = "../sn_protocol", version = "0.16.4-alpha.0" } sn_registers = { path = "../sn_registers", version = "0.3.13-alpha.0" } sn_transfers = { path = "../sn_transfers", version = "0.18.0-alpha.0" } -sn_service_management = { path = "../sn_service_management", version = "0.2.5-alpha.0" } +sn_service_management = { path = "../sn_service_management", version = "0.2.5-alpha.1" } thiserror = "1.0.23" tokio = { version = "1.32.0", features = [ "io-util", diff --git a/sn_node_manager/Cargo.toml b/sn_node_manager/Cargo.toml index dbcd880920..f816fc7548 100644 --- a/sn_node_manager/Cargo.toml +++ b/sn_node_manager/Cargo.toml @@ -7,7 +7,7 @@ license = "GPL-3.0" name = "sn-node-manager" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.7.5-alpha.2" +version = "0.7.5-alpha.3" [[bin]] name = "safenode-manager" @@ -43,7 +43,7 @@ serde_json = "1.0" service-manager = "0.6.0" sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.11-alpha.0" } sn_protocol = { path = "../sn_protocol", version = "0.16.4-alpha.0" } -sn_service_management = { path = "../sn_service_management", version = "0.2.5-alpha.0" } +sn_service_management = { path = "../sn_service_management", version = "0.2.5-alpha.1" } sn-releases = "0.2.0" sn_transfers = { path = "../sn_transfers", version = "0.18.0-alpha.0" } sysinfo = "0.30.8" diff --git a/sn_node_rpc_client/Cargo.toml b/sn_node_rpc_client/Cargo.toml index 71676764bc..8e6cbdc68a 100644 --- a/sn_node_rpc_client/Cargo.toml +++ b/sn_node_rpc_client/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" name = "sn_node_rpc_client" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.6.12-alpha.0" +version = "0.6.13-alpha.0" [[bin]] name = "safenode_rpc_client" @@ -23,12 +23,12 @@ color-eyre = "0.6.2" hex = "~0.4.3" libp2p = { version="0.53", features = ["kad"]} libp2p-identity = { version="0.2.7", features = ["rand"] } -sn_client = { path = "../sn_client", version = "0.106.0-alpha.4" } +sn_client = { path = "../sn_client", version = "0.106.0-alpha.5" } sn_logging = { path = "../sn_logging", version = "0.2.26-alpha.0" } -sn_node = { path = "../sn_node", version = "0.106.0-alpha.4" } +sn_node = { path = "../sn_node", version = "0.106.0-alpha.5" } sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.11-alpha.0" } sn_protocol = { path = "../sn_protocol", version = "0.16.4-alpha.0", features=["rpc"] } -sn_service_management = { path = "../sn_service_management", version = "0.2.5-alpha.0" } +sn_service_management = { path = "../sn_service_management", version = "0.2.5-alpha.1" } sn_transfers = { path = "../sn_transfers", version = "0.18.0-alpha.0" } thiserror = "1.0.23" # # watch out updating this, protoc compiler needs to be installed on all build systems diff --git a/sn_service_management/Cargo.toml b/sn_service_management/Cargo.toml index 46f9c6beed..14f62dd603 100644 --- a/sn_service_management/Cargo.toml +++ b/sn_service_management/Cargo.toml @@ -7,7 +7,7 @@ license = "GPL-3.0" name = "sn_service_management" readme = "README.md" repository = "https://github.com/maidsafe/safe_network" -version = "0.2.5-alpha.0" +version = "0.2.5-alpha.1" [dependencies] async-trait = "0.1" From 870ef15c8afc1cc22b08dbbfd9ce0181a36630b2 Mon Sep 17 00:00:00 2001 From: Ian Coleman Date: Tue, 26 Mar 2024 14:32:19 +1100 Subject: [PATCH 146/205] feat(cli): eip2333 helpers for accounts --- Cargo.lock | 1111 ++++++++++++++++++++------ sn_cli/Cargo.toml | 3 + sn_cli/src/acc_packet/user_secret.rs | 72 ++ 3 files changed, 954 insertions(+), 232 deletions(-) create mode 100644 sn_cli/src/acc_packet/user_secret.rs diff --git a/Cargo.lock b/Cargo.lock index 97f9f2e5f8..3f65bd07ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,7 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ "crypto-common", - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -36,7 +36,7 @@ dependencies = [ "cfg-if", "cipher 0.3.0", "cpufeatures", - "opaque-debug", + "opaque-debug 0.3.1", ] [[package]] @@ -86,7 +86,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", - "getrandom", + "getrandom 0.2.14", "once_cell", "version_check", "zerocopy", @@ -339,7 +339,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -350,7 +350,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -390,6 +390,15 @@ dependencies = [ "url", ] +[[package]] +name = "autocfg" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78" +dependencies = [ + "autocfg 1.2.0", +] + [[package]] name = "autocfg" version = "1.2.0" @@ -448,10 +457,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" dependencies = [ "futures-core", - "getrandom", + "getrandom 0.2.14", "instant", "pin-project-lite", - "rand", + "rand 0.8.5", "tokio", ] @@ -476,6 +485,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base64" version = "0.13.1" @@ -515,6 +530,17 @@ dependencies = [ "serde", ] +[[package]] +name = "bip39" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f2635620bf0b9d4576eb7bb9a38a55df78bd1205d26fa994b25911a69f212f" +dependencies = [ + "bitcoin_hashes 0.11.0", + "serde", + "unicode-normalization", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -539,10 +565,10 @@ dependencies = [ "base64 0.21.7", "bech32", "bitcoin-internals", - "bitcoin_hashes", + "bitcoin_hashes 0.13.0", "hex-conservative", "hex_lit", - "secp256k1", + "secp256k1 0.28.2", ] [[package]] @@ -551,6 +577,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9425c3bf7089c983facbae04de54513cce73b41c7f9ff8c845b54e7bc64ebbfb" +[[package]] +name = "bitcoin_hashes" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90064b8dee6815a6470d60bad07bbbaee885c0e12d04177138fa3291a01b7bc4" + [[package]] name = "bitcoin_hashes" version = "0.13.0" @@ -594,13 +626,26 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding 0.1.5", + "byte-tools", + "byteorder", + "generic-array 0.12.4", +] + [[package]] name = "block-buffer" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "generic-array", + "block-padding 0.2.1", + "generic-array 0.14.7", ] [[package]] @@ -609,7 +654,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -622,6 +667,15 @@ dependencies = [ "cipher 0.3.0", ] +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + [[package]] name = "block-padding" version = "0.2.1" @@ -634,7 +688,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -660,7 +714,7 @@ dependencies = [ "ff", "group", "pairing", - "rand_core", + "rand_core 0.6.4", "serde", "subtle", ] @@ -675,11 +729,11 @@ dependencies = [ "blstrs", "ff", "group", - "hex", + "hex 0.4.3", "hex_fmt", "pairing", - "rand", - "rand_chacha", + "rand 0.8.5", + "rand_chacha 0.3.1", "serde", "thiserror", "tiny-keccak", @@ -739,6 +793,12 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + [[package]] name = "bytemuck" version = "1.15.0" @@ -807,12 +867,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6e324229dc011159fcc089755d1e2e216a90d43a7dea6853ca740b84f35e7" +checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b" dependencies = [ "jobserver", "libc", + "once_cell", ] [[package]] @@ -847,9 +908,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.37" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", @@ -898,7 +959,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -943,7 +1004,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -952,6 +1013,15 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "color-eyre" version = "0.6.3" @@ -1179,17 +1249,39 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array 0.14.7", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array", - "rand_core", + "generic-array 0.14.7", + "rand_core 0.6.4", "typenum", ] +[[package]] +name = "crypto-mac" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25fab6889090c8133f3deb8f73ba3c65a7f456f66436fc012a1b1e272b1e103e" +dependencies = [ + "generic-array 0.14.7", + "subtle", +] + [[package]] name = "ctr" version = "0.9.2" @@ -1199,6 +1291,19 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "curve25519-dalek" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9fdf9972b2bd6af2d913799d9ebc165ea4d2e65878e329d9c6b372c4491b61" +dependencies = [ + "byteorder", + "digest 0.9.0", + "rand_core 0.5.1", + "subtle", + "zeroize", +] + [[package]] name = "curve25519-dalek" version = "4.1.2" @@ -1224,7 +1329,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -1246,7 +1351,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", "synstructure 0.13.1", ] @@ -1271,7 +1376,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.10.0", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -1282,7 +1387,7 @@ checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ "darling_core", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -1320,6 +1425,16 @@ dependencies = [ "uuid", ] +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der" version = "0.7.9" @@ -1339,7 +1454,7 @@ dependencies = [ "asn1-rs", "displaydoc", "nom", - "num-bigint", + "num-bigint 0.4.4", "num-traits", "rusticata-macros", ] @@ -1372,13 +1487,22 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array 0.12.4", +] + [[package]] name = "digest" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -1441,7 +1565,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -1477,14 +1601,26 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der 0.6.1", + "elliptic-curve", + "rfc6979", + "signature 1.6.4", +] + [[package]] name = "ed25519" version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", - "signature", + "pkcs8 0.10.2", + "signature 2.2.0", ] [[package]] @@ -1493,11 +1629,11 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" dependencies = [ - "curve25519-dalek", + "curve25519-dalek 4.1.2", "ed25519", - "rand_core", + "rand_core 0.6.4", "serde", - "sha2", + "sha2 0.10.8", "subtle", "zeroize", ] @@ -1508,6 +1644,26 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint", + "der 0.6.1", + "digest 0.10.7", + "ff", + "generic-array 0.14.7", + "group", + "pkcs8 0.9.0", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "encode_unicode" version = "0.3.6" @@ -1532,7 +1688,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -1606,6 +1762,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + [[package]] name = "fastrand" version = "2.0.2" @@ -1619,15 +1781,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" dependencies = [ "bitvec", - "rand_core", + "rand_core 0.6.4", "subtle", ] +[[package]] +name = "ff-zeroize" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c02169a2e8515aa316ce516eaaf6318a76617839fbf904073284bc2576b029ee" +dependencies = [ + "byteorder", + "ff_derive-zeroize", + "rand_core 0.5.1", + "zeroize", +] + +[[package]] +name = "ff_derive-zeroize" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b24d4059bc0d0a0bf26b740aa21af1f96a984f0ab7a21356d00b32475388b53a" +dependencies = [ + "num-bigint 0.2.6", + "num-integer", + "num-traits", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "fiat-crypto" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c007b1ae3abe1cb6f85a16305acd418b7ca6343b953633fee2b76d8f108b830f" +checksum = "38793c55593b33412e3ae40c2c9781ffaa6f438f6f8c10f24e71846fbd7ae01e" [[package]] name = "file-rotate" @@ -1719,6 +1907,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "funty" version = "2.0.0" @@ -1802,7 +1996,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -1812,7 +2006,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35bd3cf68c183738046838e300353e4716c674dc5e56890de4826801a6622a28" dependencies = [ "futures-io", - "rustls 0.21.10", + "rustls 0.21.11", ] [[package]] @@ -1866,6 +2060,15 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" +dependencies = [ + "typenum", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1876,6 +2079,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.2.14" @@ -1885,7 +2099,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -1895,7 +2109,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" dependencies = [ - "opaque-debug", + "opaque-debug 0.3.1", "polyval", ] @@ -1959,7 +2173,7 @@ dependencies = [ "into-attr-derive", "pest", "pest_derive", - "rand", + "rand 0.8.5", "tempfile", ] @@ -1970,9 +2184,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" dependencies = [ "ff", - "rand", - "rand_core", - "rand_xorshift", + "rand 0.8.5", + "rand_core 0.6.4", + "rand_xorshift 0.3.0", "subtle", ] @@ -2072,11 +2286,20 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hex" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77" + [[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +dependencies = [ + "serde", +] [[package]] name = "hex-conservative" @@ -2098,9 +2321,9 @@ checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" [[package]] name = "hickory-proto" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "091a6fbccf4860009355e3efc52ff4acf37a63489aad7435372d44ceeb6fbbcf" +checksum = "07698b8420e2f0d6447a436ba999ec85d8fbf2a398bbd737b82cac4a2e96e512" dependencies = [ "async-trait", "cfg-if", @@ -2112,7 +2335,7 @@ dependencies = [ "idna 0.4.0", "ipnet", "once_cell", - "rand", + "rand 0.8.5", "socket2", "thiserror", "tinyvec", @@ -2123,9 +2346,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35b8f021164e6a984c9030023544c57789c51760065cd510572fedcfb04164e8" +checksum = "28757f23aa75c98f254cf0405e6d8c25b831b32921b050a66692427679b1f243" dependencies = [ "cfg-if", "futures-util", @@ -2134,7 +2357,7 @@ dependencies = [ "lru-cache", "once_cell", "parking_lot", - "rand", + "rand 0.8.5", "resolv-conf", "smallvec", "thiserror", @@ -2148,7 +2371,17 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", +] + +[[package]] +name = "hmac" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +dependencies = [ + "crypto-mac", + "digest 0.9.0", ] [[package]] @@ -2274,9 +2507,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186548d73ac615b32a73aafe38fb4f56c0d340e110e5a200bcadbaf2e199263a" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" dependencies = [ "bytes", "futures-channel", @@ -2300,7 +2533,7 @@ dependencies = [ "futures-util", "http 0.2.12", "hyper 0.14.28", - "rustls 0.21.10", + "rustls 0.21.11", "tokio", "tokio-rustls 0.24.1", ] @@ -2313,9 +2546,9 @@ checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.2.0", + "hyper 1.3.1", "hyper-util", - "rustls 0.22.3", + "rustls 0.22.4", "rustls-pki-types", "tokio", "tokio-rustls 0.25.0", @@ -2345,7 +2578,7 @@ dependencies = [ "futures-util", "http 1.1.0", "http-body 1.0.0", - "hyper 1.2.0", + "hyper 1.3.1", "pin-project-lite", "socket2", "tokio", @@ -2445,7 +2678,7 @@ dependencies = [ "http 0.2.12", "hyper 0.14.28", "log", - "rand", + "rand 0.8.5", "tokio", "url", "xmltree", @@ -2479,7 +2712,7 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ - "autocfg", + "autocfg 1.2.0", "hashbrown 0.12.3", ] @@ -2532,7 +2765,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" dependencies = [ "block-padding 0.3.3", - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -2624,9 +2857,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "685a7d121ee3f65ae4fddd72b25a04bb36b6af81bc0828f7d5434c0fe60fa3a2" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" dependencies = [ "libc", ] @@ -2640,6 +2873,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "keccak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +dependencies = [ + "cpufeatures", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -2668,7 +2910,7 @@ dependencies = [ "either", "futures", "futures-timer", - "getrandom", + "getrandom 0.2.14", "instant", "libp2p-allow-block-list", "libp2p-connection-limits", @@ -2740,7 +2982,7 @@ dependencies = [ "parking_lot", "pin-project", "quick-protobuf", - "rand", + "rand 0.8.5", "rw-stream-sink", "smallvec", "thiserror", @@ -2802,7 +3044,7 @@ dependencies = [ "fnv", "futures", "futures-ticker", - "getrandom", + "getrandom 0.2.14", "hex_fmt", "instant", "libp2p-core", @@ -2811,9 +3053,9 @@ dependencies = [ "prometheus-client", "quick-protobuf", "quick-protobuf-codec 0.3.1", - "rand", + "rand 0.8.5", "regex", - "sha2", + "sha2 0.10.8", "smallvec", "tracing", "void", @@ -2853,8 +3095,8 @@ dependencies = [ "hkdf", "multihash", "quick-protobuf", - "rand", - "sha2", + "rand 0.8.5", + "sha2 0.10.8", "thiserror", "tracing", "zeroize", @@ -2880,8 +3122,8 @@ dependencies = [ "libp2p-swarm", "quick-protobuf", "quick-protobuf-codec 0.3.1", - "rand", - "sha2", + "rand 0.8.5", + "sha2 0.10.8", "smallvec", "thiserror", "tracing", @@ -2902,7 +3144,7 @@ dependencies = [ "libp2p-core", "libp2p-identity", "libp2p-swarm", - "rand", + "rand 0.8.5", "smallvec", "socket2", "tokio", @@ -2937,7 +3179,7 @@ checksum = "8ecd0545ce077f6ea5434bcb76e8d0fe942693b4380aaad0d34a358c2bd05793" dependencies = [ "asynchronous-codec 0.7.0", "bytes", - "curve25519-dalek", + "curve25519-dalek 4.1.2", "futures", "libp2p-core", "libp2p-identity", @@ -2945,8 +3187,8 @@ dependencies = [ "multihash", "once_cell", "quick-protobuf", - "rand", - "sha2", + "rand 0.8.5", + "sha2 0.10.8", "snow", "static_assertions", "thiserror", @@ -2970,9 +3212,9 @@ dependencies = [ "libp2p-tls", "parking_lot", "quinn", - "rand", + "rand 0.8.5", "ring 0.16.20", - "rustls 0.21.10", + "rustls 0.21.11", "socket2", "thiserror", "tokio", @@ -2997,7 +3239,7 @@ dependencies = [ "libp2p-swarm", "quick-protobuf", "quick-protobuf-codec 0.3.1", - "rand", + "rand 0.8.5", "static_assertions", "thiserror", "tracing", @@ -3019,7 +3261,7 @@ dependencies = [ "libp2p-core", "libp2p-identity", "libp2p-swarm", - "rand", + "rand 0.8.5", "serde", "smallvec", "tracing", @@ -3036,14 +3278,14 @@ dependencies = [ "fnv", "futures", "futures-timer", - "getrandom", + "getrandom 0.2.14", "instant", "libp2p-core", "libp2p-identity", "libp2p-swarm-derive", "multistream-select", "once_cell", - "rand", + "rand 0.8.5", "smallvec", "tokio", "tracing", @@ -3060,7 +3302,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -3092,7 +3334,7 @@ dependencies = [ "libp2p-identity", "rcgen", "ring 0.16.20", - "rustls 0.21.10", + "rustls 0.21.11", "rustls-webpki 0.101.7", "thiserror", "x509-parser", @@ -3202,7 +3444,7 @@ version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ - "autocfg", + "autocfg 1.2.0", "scopeguard", ] @@ -3275,6 +3517,15 @@ dependencies = [ "libc", ] +[[package]] +name = "merkle-cbt" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171d2f700835121c3b04ccf0880882987a050fd5c7ae88148abf537d33dd3a56" +dependencies = [ + "cfg-if", +] + [[package]] name = "mime" version = "0.3.17" @@ -3314,7 +3565,7 @@ checksum = "00a000cf8bbbfb123a9bdc66b61c2885a4bb038df4f2629884caafabeb76b0f9" dependencies = [ "log", "once_cell", - "rustls 0.21.10", + "rustls 0.21.11", "rustls-webpki 0.101.7", "webpki-roots 0.25.4", ] @@ -3326,7 +3577,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.48.0", ] @@ -3381,7 +3632,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -3608,7 +3859,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3135b08af27d103b0a51f2ae0f8632117b7b185ccf931445affa8df530576a41" dependencies = [ - "num-bigint", + "num-bigint 0.4.4", "num-complex", "num-integer", "num-iter", @@ -3616,13 +3867,24 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg 1.2.0", + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" dependencies = [ - "autocfg", + "autocfg 1.2.0", "num-integer", "num-traits", "serde", @@ -3669,7 +3931,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" dependencies = [ - "autocfg", + "autocfg 1.2.0", "num-integer", "num-traits", ] @@ -3680,8 +3942,8 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" dependencies = [ - "autocfg", - "num-bigint", + "autocfg 1.2.0", + "num-bigint 0.4.4", "num-integer", "num-traits", "serde", @@ -3693,7 +3955,7 @@ version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ - "autocfg", + "autocfg 1.2.0", "libm", ] @@ -3752,6 +4014,12 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -3839,7 +4107,7 @@ dependencies = [ "opentelemetry_api", "ordered-float", "percent-encoding", - "rand", + "rand 0.8.5", "regex", "serde_json", "thiserror", @@ -3868,6 +4136,17 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2 0.10.8", +] + [[package]] name = "pairing" version = "0.22.0" @@ -3877,6 +4156,21 @@ dependencies = [ "group", ] +[[package]] +name = "pairing-plus" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58cda4f22e8e6720f3c254049960c8cc4f93cb82b5ade43bddd2622b5f39ea62" +dependencies = [ + "byteorder", + "digest 0.8.1", + "ff-zeroize", + "rand 0.4.6", + "rand_core 0.5.1", + "rand_xorshift 0.2.0", + "zeroize", +] + [[package]] name = "parking" version = "2.2.0" @@ -3913,7 +4207,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" dependencies = [ "base64ct", - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -3930,9 +4224,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" dependencies = [ "digest 0.10.7", - "hmac", + "hmac 0.12.1", "password-hash", - "sha2", + "sha2 0.10.8", ] [[package]] @@ -3982,7 +4276,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -3993,7 +4287,7 @@ checksum = "2adbf29bb9776f28caece835398781ab24435585fe0d4dc1374a61db5accedca" dependencies = [ "once_cell", "pest", - "sha2", + "sha2 0.10.8", ] [[package]] @@ -4025,7 +4319,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -4040,14 +4334,24 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der 0.6.1", + "spki 0.6.0", +] + [[package]] name = "pkcs8" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.9", + "spki 0.7.3", ] [[package]] @@ -4106,9 +4410,9 @@ dependencies = [ [[package]] name = "polling" -version = "3.6.0" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c976a60b2d7e99d6f229e414670a9b85d13ac305cc6d1e9c134de58c5aaaf6" +checksum = "645493cf344456ef24219d02a768cf1fb92ddf8c92161679ae3d91b91a637be3" dependencies = [ "cfg-if", "concurrent-queue", @@ -4126,7 +4430,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ "cpufeatures", - "opaque-debug", + "opaque-debug 0.3.1", "universal-hash", ] @@ -4138,7 +4442,7 @@ checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", "cpufeatures", - "opaque-debug", + "opaque-debug 0.3.1", "universal-hash", ] @@ -4251,9 +4555,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.80" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56dea16b0a29e94408b9aa5e2940a4eedbd128a1ba20e8f7ae60fd3d465af0e" +checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" dependencies = [ "unicode-ident", ] @@ -4278,7 +4582,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -4292,9 +4596,9 @@ dependencies = [ "bitflags 2.5.0", "lazy_static", "num-traits", - "rand", - "rand_chacha", - "rand_xorshift", + "rand 0.8.5", + "rand_chacha 0.3.1", + "rand_xorshift 0.3.0", "regex-syntax 0.8.3", "rusty-fork", "tempfile", @@ -4444,7 +4748,7 @@ checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" dependencies = [ "env_logger", "log", - "rand", + "rand 0.8.5", ] [[package]] @@ -4459,7 +4763,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.21.10", + "rustls 0.21.11", "thiserror", "tokio", "tracing", @@ -4472,10 +4776,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "141bf7dfde2fbc246bfd3fe12f2455aa24b0fbd9af535d8c86c7bd1381ff2b1a" dependencies = [ "bytes", - "rand", + "rand 0.8.5", "ring 0.16.20", "rustc-hash", - "rustls 0.21.10", + "rustls 0.21.11", "slab", "thiserror", "tinyvec", @@ -4510,6 +4814,51 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.8", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc 0.1.0", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift 0.1.1", + "winapi", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + [[package]] name = "rand" version = "0.8.5" @@ -4517,8 +4866,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.8", + "rand_core 0.3.1", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", ] [[package]] @@ -4528,7 +4897,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", ] [[package]] @@ -4537,7 +4930,87 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.14", +] + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.8", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_xorshift" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77d416b86801d23dde1aa643023b775c3a462efc0ed96443add11546cdf1dca8" +dependencies = [ + "rand_core 0.5.1", ] [[package]] @@ -4546,7 +5019,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" dependencies = [ - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -4581,6 +5054,15 @@ dependencies = [ "yasna", ] +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -4596,7 +5078,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" dependencies = [ - "getrandom", + "getrandom 0.2.14", "libredox", "thiserror", ] @@ -4668,7 +5150,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.10", + "rustls 0.21.11", "rustls-pemfile 1.0.4", "serde", "serde_json", @@ -4688,9 +5170,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e6cc1e89e689536eb5aeede61520e874df5a4707df811cd5da4aa5fbb2aae19" +checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" dependencies = [ "base64 0.22.0", "bytes", @@ -4699,7 +5181,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.2.0", + "hyper 1.3.1", "hyper-rustls 0.26.0", "hyper-util", "ipnet", @@ -4709,7 +5191,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.22.3", + "rustls 0.22.4", "rustls-pemfile 2.1.2", "rustls-pki-types", "serde", @@ -4737,6 +5219,17 @@ dependencies = [ "quick-error", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint", + "hmac 0.12.1", + "zeroize", +] + [[package]] name = "rgb" version = "0.8.37" @@ -4769,7 +5262,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.14", "libc", "spin 0.9.8", "untrusted 0.9.0", @@ -4778,9 +5271,9 @@ dependencies = [ [[package]] name = "rmp" -version = "0.8.12" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9860a6cc38ed1da53456442089b4dfa35e7cedaa326df63017af88385e6b20" +checksum = "228ed7c16fa39782c3b3468e974aec2795e9089153cd08ee2e9aefb3613334c4" dependencies = [ "byteorder", "num-traits", @@ -4789,9 +5282,9 @@ dependencies = [ [[package]] name = "rmp-serde" -version = "1.1.2" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bffea85eea980d8a74453e5d02a8d93028f3c34725de143085a844ebe953258a" +checksum = "938a142ab806f18b88a97b0dea523d39e0fd730a064b035726adcfc58a8a5188" dependencies = [ "byteorder", "rmp", @@ -4845,9 +5338,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.32" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ "bitflags 2.5.0", "errno", @@ -4883,9 +5376,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.10" +version = "0.21.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" dependencies = [ "log", "ring 0.17.8", @@ -4895,14 +5388,14 @@ dependencies = [ [[package]] name = "rustls" -version = "0.22.3" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99008d7ad0bbbea527ec27bddbc0e432c5b87d8175178cee68d2eec9c4a1813c" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" dependencies = [ "log", "ring 0.17.8", "rustls-pki-types", - "rustls-webpki 0.102.2", + "rustls-webpki 0.102.3", "subtle", "zeroize", ] @@ -4937,9 +5430,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd36cc4259e3e4514335c4a138c6b43171a8d61d8f5c9348f9fc7529416f247" +checksum = "beb461507cee2c2ff151784c52762cf4d9ff6a61f3e80968600ed24fa837fa54" [[package]] name = "rustls-webpki" @@ -4953,9 +5446,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.2" +version = "0.102.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" +checksum = "f3bce581c0dd41bce533ce695a1437fa16a7ab5ac3ccfa99fe1a620a7885eabf" dependencies = [ "ring 0.17.8", "rustls-pki-types", @@ -5038,15 +5531,49 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der 0.6.1", + "generic-array 0.14.7", + "pkcs8 0.9.0", + "subtle", + "zeroize", +] + +[[package]] +name = "secp256k1" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d03ceae636d0fed5bae6a7f4f664354c5f4fcedf6eef053fef17e49f837d0a" +dependencies = [ + "rand 0.6.5", + "secp256k1-sys 0.4.2", + "serde", +] + [[package]] name = "secp256k1" version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d24b59d129cdadea20aea4fb2352fa053712e5d713eee47d700cd4b2bc002f10" dependencies = [ - "bitcoin_hashes", - "rand", - "secp256k1-sys", + "bitcoin_hashes 0.13.0", + "rand 0.8.5", + "secp256k1-sys 0.9.2", +] + +[[package]] +name = "secp256k1-sys" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957da2573cde917463ece3570eab4a0b3f19de6f1646cde62e6fd3868f566036" +dependencies = [ + "cc", ] [[package]] @@ -5060,9 +5587,9 @@ dependencies = [ [[package]] name = "self_encryption" -version = "0.29.1" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab2cd87e583738aba86278972e9116e2aabdb7fceda2be1fb3abe543be2336e" +checksum = "894da3241a9e426c16fb8cb28b19416eae5fafdc7742e4bc505c1821661c140f" dependencies = [ "aes 0.8.4", "bincode", @@ -5070,11 +5597,11 @@ dependencies = [ "bytes", "cbc", "err-derive", - "hex", + "hex 0.4.3", "itertools 0.10.5", "num_cpus", - "rand", - "rand_chacha", + "rand 0.8.5", + "rand_chacha 0.3.1", "rayon", "serde", "tempfile", @@ -5103,29 +5630,38 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.197" +version = "1.0.198" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc" dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b8497c313fd43ab992087548117643f6fcd935cbf36f176ffda0aacf9591734" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" -version = "1.0.197" +version = "1.0.198" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] name = "serde_json" -version = "1.0.115" +version = "1.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" +checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" dependencies = [ "itoa", "ryu", @@ -5189,7 +5725,7 @@ dependencies = [ "cfg-if", "cpufeatures", "digest 0.9.0", - "opaque-debug", + "opaque-debug 0.3.1", ] [[package]] @@ -5203,6 +5739,31 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha2" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" +dependencies = [ + "block-buffer 0.7.3", + "digest 0.8.1", + "fake-simd", + "opaque-debug 0.2.3", +] + +[[package]] +name = "sha2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpufeatures", + "digest 0.9.0", + "opaque-debug 0.3.1", +] + [[package]] name = "sha2" version = "0.10.8" @@ -5214,6 +5775,18 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha3" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81199417d4e5de3f04b1e871023acea7389672c4135918f05aa9cbf2f2fa809" +dependencies = [ + "block-buffer 0.9.0", + "digest 0.9.0", + "keccak", + "opaque-debug 0.3.1", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -5231,20 +5804,30 @@ checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + [[package]] name = "signature" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -5253,7 +5836,7 @@ version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ - "autocfg", + "autocfg 1.2.0", ] [[package]] @@ -5281,7 +5864,7 @@ dependencies = [ "nix 0.27.1", "predicates 3.1.0", "prost 0.9.0", - "reqwest 0.12.3", + "reqwest 0.12.4", "semver", "serde", "serde_json", @@ -5338,6 +5921,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "sn_bls_ckd" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cc1905b0d5c8c8dd4cfafa1b064645e8eb57c26ad93a491acbaa2dc59c3d8c2" +dependencies = [ + "hex 0.3.2", + "hkdf", + "sha2 0.10.8", + "sn_curv", +] + [[package]] name = "sn_build_info" version = "0.1.7" @@ -5351,6 +5946,7 @@ version = "0.90.4" dependencies = [ "aes 0.7.5", "base64 0.22.0", + "bip39", "bitcoin", "block-modes", "blsttc", @@ -5364,16 +5960,18 @@ dependencies = [ "dirs-next", "eyre", "futures", - "hex", + "hex 0.4.3", "indicatif", "libp2p", - "rand", + "rand 0.8.5", "rayon", - "reqwest 0.12.3", + "reqwest 0.12.4", "rmp-serde", "serde", + "sn_bls_ckd", "sn_build_info", "sn_client", + "sn_curv", "sn_logging", "sn_peers_acquisition", "sn_protocol", @@ -5401,15 +5999,15 @@ dependencies = [ "dirs-next", "eyre", "futures", - "getrandom", - "hex", + "getrandom 0.2.14", + "hex 0.4.3", "itertools 0.12.1", "lazy_static", "libp2p", "libp2p-identity", "petgraph", "prometheus-client", - "rand", + "rand 0.8.5", "rayon", "rmp-serde", "self_encryption", @@ -5435,6 +6033,39 @@ dependencies = [ "xor_name", ] +[[package]] +name = "sn_curv" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7b53f6b7e77c36e00b5469e77386c11d5a8d863300acb4bd373227894e3a117" +dependencies = [ + "curve25519-dalek 3.2.0", + "digest 0.9.0", + "ff-zeroize", + "generic-array 0.14.7", + "hex 0.4.3", + "hmac 0.11.0", + "lazy_static", + "merkle-cbt", + "num-bigint 0.4.4", + "num-integer", + "num-traits", + "p256", + "pairing-plus", + "rand 0.6.5", + "rand 0.7.3", + "secp256k1 0.20.3", + "serde", + "serde_bytes", + "serde_derive", + "sha2 0.8.2", + "sha2 0.9.9", + "sha3", + "thiserror", + "typenum", + "zeroize", +] + [[package]] name = "sn_faucet" version = "0.4.9" @@ -5447,7 +6078,7 @@ dependencies = [ "color-eyre", "dirs-next", "fs2", - "hex", + "hex 0.4.3", "indicatif", "minreq", "serde", @@ -5474,7 +6105,7 @@ dependencies = [ "opentelemetry", "opentelemetry-otlp", "opentelemetry-semantic-conventions", - "rand", + "rand 0.8.5", "serde", "serde_json", "sysinfo", @@ -5514,15 +6145,15 @@ dependencies = [ "custom_debug", "eyre", "futures", - "getrandom", - "hex", + "getrandom 0.2.14", + "hex 0.4.3", "hyper 0.14.28", "itertools 0.12.1", "libp2p", "libp2p-identity", "prometheus-client", "quickcheck", - "rand", + "rand 0.8.5", "rayon", "rmp-serde", "serde", @@ -5560,15 +6191,15 @@ dependencies = [ "eyre", "file-rotate", "futures", - "hex", + "hex 0.4.3", "itertools 0.12.1", "lazy_static", "libp2p", "prometheus-client", "prost 0.9.0", - "rand", + "rand 0.8.5", "rayon", - "reqwest 0.12.3", + "reqwest 0.12.4", "rmp-serde", "self_encryption", "serde", @@ -5606,7 +6237,7 @@ dependencies = [ "blsttc", "clap", "color-eyre", - "hex", + "hex 0.4.3", "libp2p", "libp2p-identity", "sn_client", @@ -5631,8 +6262,8 @@ dependencies = [ "clap", "lazy_static", "libp2p", - "rand", - "reqwest 0.12.3", + "rand 0.8.5", + "reqwest 0.12.4", "sn_protocol", "thiserror", "tokio", @@ -5650,14 +6281,14 @@ dependencies = [ "crdts", "custom_debug", "dirs-next", - "hex", + "hex 0.4.3", "lazy_static", "libp2p", "prost 0.9.0", "rmp-serde", "serde", "serde_json", - "sha2", + "sha2 0.10.8", "sn_build_info", "sn_registers", "sn_transfers", @@ -5676,9 +6307,9 @@ dependencies = [ "blsttc", "crdts", "eyre", - "hex", + "hex 0.4.3", "proptest", - "rand", + "rand 0.8.5", "rmp-serde", "serde", "thiserror", @@ -5721,11 +6352,11 @@ dependencies = [ "dirs-next", "eyre", "fs2", - "hex", + "hex 0.4.3", "lazy_static", "libp2p", "pprof", - "rand", + "rand 0.8.5", "rayon", "rmp-serde", "serde", @@ -5747,11 +6378,11 @@ dependencies = [ "aes-gcm", "blake2", "chacha20poly1305", - "curve25519-dalek", - "rand_core", + "curve25519-dalek 4.1.2", + "rand_core 0.6.4", "ring 0.17.8", "rustc_version", - "sha2", + "sha2 0.10.8", "subtle", ] @@ -5776,7 +6407,7 @@ dependencies = [ "futures", "httparse", "log", - "rand", + "rand 0.8.5", "sha-1", ] @@ -5792,6 +6423,16 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der 0.6.1", +] + [[package]] name = "spki" version = "0.7.3" @@ -5799,7 +6440,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.9", ] [[package]] @@ -5851,7 +6492,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -5896,9 +6537,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.58" +version = "2.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" +checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" dependencies = [ "proc-macro2", "quote", @@ -5931,14 +6572,14 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] name = "sysinfo" -version = "0.30.10" +version = "0.30.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d7c217777061d5a2d652aea771fb9ba98b6dade657204b08c4b9604d11555b" +checksum = "87341a165d73787554941cd5ef55ad728011566fe714e987d1b976c15dbc3a83" dependencies = [ "cfg-if", "core-foundation-sys", @@ -6018,22 +6659,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" +checksum = "f0126ad08bff79f29fc3ae6a55cc72352056dfff61e3ff8bb7129476d44b23aa" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" +checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -6186,7 +6827,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -6206,7 +6847,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.10", + "rustls 0.21.11", "tokio", ] @@ -6216,7 +6857,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" dependencies = [ - "rustls 0.22.3", + "rustls 0.22.4", "rustls-pki-types", "tokio", ] @@ -6355,7 +6996,7 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand", + "rand 0.8.5", "slab", "tokio", "tokio-util 0.7.10", @@ -6408,7 +7049,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -6562,7 +7203,7 @@ dependencies = [ "http 1.1.0", "httparse", "log", - "rand", + "rand 0.8.5", "sha1", "thiserror", "url", @@ -6589,7 +7230,7 @@ checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52" dependencies = [ "byteorder", "crunchy", - "hex", + "hex 0.4.3", "static_assertions", ] @@ -6622,9 +7263,9 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" dependencies = [ "tinyvec", ] @@ -6736,7 +7377,7 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" dependencies = [ - "getrandom", + "getrandom 0.2.14", ] [[package]] @@ -6826,6 +7467,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -6853,7 +7500,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", "wasm-bindgen-shared", ] @@ -6887,7 +7534,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6993,11 +7640,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "134306a13c5647ad6453e8deaec55d3a44d6021970129e6188735e74bf546697" dependencies = [ - "winapi", + "windows-sys 0.52.0", ] [[package]] @@ -7218,8 +7865,8 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" dependencies = [ - "curve25519-dalek", - "rand_core", + "curve25519-dalek 4.1.2", + "rand_core 0.6.4", "serde", "zeroize", ] @@ -7273,9 +7920,9 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fd9dddecfdbc7c17ae93da6d28a5a9c4f5564abe7b735d2530c7a159b6b55e8" dependencies = [ - "hex", - "rand", - "rand_core", + "hex 0.4.3", + "rand 0.8.5", + "rand_core 0.6.4", "serde", "serde_test", "tiny-keccak", @@ -7292,7 +7939,7 @@ dependencies = [ "nohash-hasher", "parking_lot", "pin-project", - "rand", + "rand 0.8.5", "static_assertions", ] @@ -7308,7 +7955,7 @@ dependencies = [ "nohash-hasher", "parking_lot", "pin-project", - "rand", + "rand 0.8.5", "static_assertions", ] @@ -7338,7 +7985,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -7358,7 +8005,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.60", ] [[package]] @@ -7374,7 +8021,7 @@ dependencies = [ "crc32fast", "crossbeam-utils", "flate2", - "hmac", + "hmac 0.12.1", "pbkdf2", "sha1", "time", diff --git a/sn_cli/Cargo.toml b/sn_cli/Cargo.toml index d0cf2e521d..10d767dc0f 100644 --- a/sn_cli/Cargo.toml +++ b/sn_cli/Cargo.toml @@ -33,6 +33,8 @@ open-metrics = ["sn_client/open-metrics"] [dependencies] aes="0.7.5" base64 = { version = "0.22.0", optional = true } +bip39 = "2.0.0" +curv = { version = "0.10.1", package = "sn_curv", default-features = false, features = ["num-bigint"] } bitcoin = { version = "0.31.0", optional = true} block-modes="0.8.1" bls = { package = "blsttc", version = "8.0.1" } @@ -43,6 +45,7 @@ clap = { version = "4.2.1", features = ["derive"]} color-eyre = "~0.6" dialoguer = "~0.11.0" dirs-next = "~2.0.0" +eip2333 = { version = "0.2.1", package = "sn_bls_ckd" } futures = "~0.3.13" hex = "~0.4.3" indicatif = { version = "0.17.5", features = ["tokio"] } diff --git a/sn_cli/src/acc_packet/user_secret.rs b/sn_cli/src/acc_packet/user_secret.rs new file mode 100644 index 0000000000..bdef9595bd --- /dev/null +++ b/sn_cli/src/acc_packet/user_secret.rs @@ -0,0 +1,72 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use bls::SecretKey; +use color_eyre::{eyre::eyre, Result}; +use curv::elliptic::curves::ECScalar; +use rand::RngCore; +use sn_client::transfers::MainSecretKey; +use std::path::Path; +use xor_name::XorName; + +const MNEMONIC_FILENAME: &str = "account_secret"; + +const ACCOUNT_ROOT_XORNAME_DERIVATION: &str = "m/1/0"; + +const ACCOUNT_WALLET_DERIVATION: &str = "m/2/0"; + +pub(super) fn random_eip2333_mnemonic() -> Result { + let mut entropy = [1u8; 32]; + let rng = &mut rand::rngs::OsRng; + rng.fill_bytes(&mut entropy); + let mnemonic = bip39::Mnemonic::from_entropy(&entropy)?; + Ok(mnemonic) +} + +/// Derive a wallet secret key from the mnemonic for the account. +pub(super) fn account_wallet_secret_key( + mnemonic: bip39::Mnemonic, + passphrase: &str, +) -> Result { + let seed = mnemonic.to_seed(passphrase); + + let root_sk = eip2333::derive_master_sk(&seed) + .map_err(|err| eyre!("Invalid seed from mnemonic: {err}"))?; + let derived_key = eip2333::derive_child_sk(root_sk, ACCOUNT_WALLET_DERIVATION); + let key_bytes = derived_key.serialize(); + let sk = + SecretKey::from_bytes(key_bytes.into()).map_err(|err| eyre!("Invalid key bytes: {err}"))?; + Ok(MainSecretKey::new(sk)) +} + +/// Derive an xorname from the mnemonic for the account to store data. +pub(super) fn account_root_xorname(mnemonic: bip39::Mnemonic, passphrase: &str) -> Result { + let seed = mnemonic.to_seed(passphrase); + + let root_sk = eip2333::derive_master_sk(&seed) + .map_err(|err| eyre!("Invalid seed from mnemonic: {err}"))?; + let derived_key = eip2333::derive_child_sk(root_sk, ACCOUNT_ROOT_XORNAME_DERIVATION); + let derived_key_bytes = derived_key.serialize(); + Ok(XorName::from_content(&derived_key_bytes)) +} + +pub(super) fn write_mnemonic_to_disk(files_dir: &Path, mnemonic: &bip39::Mnemonic) -> Result<()> { + let filename = files_dir.join(MNEMONIC_FILENAME); + let content = mnemonic.to_string(); + std::fs::write(filename, content)?; + Ok(()) +} + +pub(super) fn read_mnemonic_from_disk(files_dir: &Path) -> Result { + let filename = files_dir.join(MNEMONIC_FILENAME); + let content = std::fs::read_to_string(filename) + .map_err(|err| eyre!("Error reading account secret: {err}"))?; + let mnemonic = bip39::Mnemonic::parse_normalized(&content) + .map_err(|err| eyre!("Error parsing account secret: {err}"))?; + Ok(mnemonic) +} From cd1c655f0f14173a0f9a1bb51da110517f9876ba Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Mon, 22 Apr 2024 10:34:20 +0900 Subject: [PATCH 147/205] feat(transfers): do not genereate wallet by default --- sn_transfers/src/wallet/error.rs | 3 +++ sn_transfers/src/wallet/hot_wallet.rs | 30 ++++++++++++++++++++------- sn_transfers/src/wallet/keys.rs | 12 +++++------ 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/sn_transfers/src/wallet/error.rs b/sn_transfers/src/wallet/error.rs index f35a7d1403..06348e11df 100644 --- a/sn_transfers/src/wallet/error.rs +++ b/sn_transfers/src/wallet/error.rs @@ -48,6 +48,9 @@ pub enum Error { /// Main pub key not found when loading wallet from path #[error("Main pub key not found: {0:#?}")] PubkeyNotFound(std::path::PathBuf), + /// Main secret key not found when loading wallet from path + #[error("Main secret key not found: {0:#?}")] + MainSecretKeyNotFound(std::path::PathBuf), /// Failed to parse bytes into a bls key #[error("Failed to parse bls key")] FailedToParseBlsKey, diff --git a/sn_transfers/src/wallet/hot_wallet.rs b/sn_transfers/src/wallet/hot_wallet.rs index 7d738952d1..ddfce397ff 100644 --- a/sn_transfers/src/wallet/hot_wallet.rs +++ b/sn_transfers/src/wallet/hot_wallet.rs @@ -9,7 +9,7 @@ use super::{ api::{WalletApi, WALLET_DIR_NAME}, data_payments::{PaymentDetails, PaymentQuote}, - keys::{get_main_key, store_new_keypair}, + keys::{get_main_key_from_disk, store_new_keypair}, wallet_file::{ get_unconfirmed_spend_requests, load_created_cash_note, remove_cash_notes, remove_unconfirmed_spend_requests, store_created_cash_notes, @@ -591,15 +591,31 @@ impl HotWallet { } /// Loads a serialized wallet from a path. + // TODO: what's the behaviour here if path has stored key and we pass one in? fn load_from_path_and_key(wallet_dir: &Path, main_key: Option) -> Result { - let key = match get_main_key(wallet_dir)? { - Some(key) => key, - None => { - let key = main_key.unwrap_or(MainSecretKey::random()); - store_new_keypair(wallet_dir, &key)?; - warn!("No main key found when loading wallet from path, generating a new one with pubkey: {:?}", key.main_pubkey()); + let key = match get_main_key_from_disk(wallet_dir) { + Ok(key) => { + if let Some(passed_key) = main_key { + if key.secret_key() != passed_key.secret_key() { + warn!("main_key passed to load_from_path_and_key, but a key was found in the wallet dir. Using the one found in the wallet dir."); + } + } + key } + Err(error) => { + if let Some(key) = main_key { + store_new_keypair(wallet_dir, &key)?; + key + } else { + error!( + "No main key found when loading wallet from path {:?}", + wallet_dir + ); + + return Err(error); + } + } }; let unconfirmed_spend_requests = match get_unconfirmed_spend_requests(wallet_dir)? { Some(unconfirmed_spend_requests) => unconfirmed_spend_requests, diff --git a/sn_transfers/src/wallet/keys.rs b/sn_transfers/src/wallet/keys.rs index ad1e4420ad..6006ac44a4 100644 --- a/sn_transfers/src/wallet/keys.rs +++ b/sn_transfers/src/wallet/keys.rs @@ -26,17 +26,17 @@ pub(crate) fn store_new_keypair(wallet_dir: &Path, main_key: &MainSecretKey) -> Ok(()) } -/// Returns Some(sn_transfers::MainSecretKey) or None if file doesn't exist. It assumes it's hex-encoded. -pub(super) fn get_main_key(wallet_dir: &Path) -> Result> { +/// Returns sn_transfers::MainSecretKey or None if file doesn't exist. It assumes it's hex-encoded. +pub(super) fn get_main_key_from_disk(wallet_dir: &Path) -> Result { let path = wallet_dir.join(MAIN_SECRET_KEY_FILENAME); if !path.is_file() { - return Ok(None); + return Err(Error::MainSecretKeyNotFound(path)); } let secret_hex_bytes = std::fs::read(&path)?; let secret = bls_secret_from_hex(secret_hex_bytes)?; - Ok(Some(MainSecretKey::new(secret))) + Ok(MainSecretKey::new(secret)) } /// Writes the public address (hex-encoded) to disk. @@ -73,7 +73,7 @@ pub fn bls_secret_from_hex>(hex: T) -> Result { #[cfg(test)] mod test { - use super::{get_main_key, store_new_keypair, MainSecretKey}; + use super::{get_main_key_from_disk, store_new_keypair, MainSecretKey}; use assert_fs::TempDir; use eyre::Result; @@ -83,7 +83,7 @@ mod test { let dir = create_temp_dir(); let root_dir = dir.path().to_path_buf(); store_new_keypair(&root_dir, &main_key)?; - let secret_result = get_main_key(&root_dir)?.expect("There to be a key on disk."); + let secret_result = get_main_key_from_disk(&root_dir)?.expect("There to be a key on disk."); assert_eq!(secret_result.main_pubkey(), main_key.main_pubkey()); Ok(()) } From d1e18d2d3431b73e22754e07fc0b1753b3e8f180 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Mon, 22 Apr 2024 11:11:49 +0900 Subject: [PATCH 148/205] feat(cli): generate a mnemonic as wallet basis if no wallet found --- sn_cli/src/acc_packet.rs | 3 +- sn_cli/src/acc_packet/user_secret.rs | 10 +++-- sn_cli/src/bin/main.rs | 2 +- sn_cli/src/bin/subcommands/wallet.rs | 7 ---- .../src/bin/subcommands/wallet/hot_wallet.rs | 37 +++++++++++++++++-- sn_cli/src/lib.rs | 1 + sn_transfers/src/genesis.rs | 6 ++- sn_transfers/src/wallet/keys.rs | 2 +- 8 files changed, 50 insertions(+), 18 deletions(-) diff --git a/sn_cli/src/acc_packet.rs b/sn_cli/src/acc_packet.rs index ec20c5ac72..d32a12ef21 100644 --- a/sn_cli/src/acc_packet.rs +++ b/sn_cli/src/acc_packet.rs @@ -7,6 +7,7 @@ // permissions and limitations relating to use of the SAFE Network Software. mod change_tracking; +pub mod user_secret; use change_tracking::*; @@ -58,7 +59,7 @@ const ACC_PACKET_OWNER_DERIVATION_INDEX: DerivationIndex = DerivationIndex([0x1; /// and tools necessary to keep an instance tracking a local storage path, as well as keeping it in sync /// with its remote version stored on the network. /// A `Client` and a the location for a funded local hot-wallet are required by this object in order to be able to connect -/// to the network, paying for data storage, and upload/retrieve information to/from the network. +/// to the network, paying for data storage, and upload/retrieve information to/from the network. /// /// TODO: currently only files and folders are supported, wallets, keys, etc., to be added later. /// diff --git a/sn_cli/src/acc_packet/user_secret.rs b/sn_cli/src/acc_packet/user_secret.rs index bdef9595bd..50724ba7a6 100644 --- a/sn_cli/src/acc_packet/user_secret.rs +++ b/sn_cli/src/acc_packet/user_secret.rs @@ -20,7 +20,7 @@ const ACCOUNT_ROOT_XORNAME_DERIVATION: &str = "m/1/0"; const ACCOUNT_WALLET_DERIVATION: &str = "m/2/0"; -pub(super) fn random_eip2333_mnemonic() -> Result { +pub fn random_eip2333_mnemonic() -> Result { let mut entropy = [1u8; 32]; let rng = &mut rand::rngs::OsRng; rng.fill_bytes(&mut entropy); @@ -29,7 +29,7 @@ pub(super) fn random_eip2333_mnemonic() -> Result { } /// Derive a wallet secret key from the mnemonic for the account. -pub(super) fn account_wallet_secret_key( +pub fn account_wallet_secret_key( mnemonic: bip39::Mnemonic, passphrase: &str, ) -> Result { @@ -44,8 +44,9 @@ pub(super) fn account_wallet_secret_key( Ok(MainSecretKey::new(sk)) } +#[allow(dead_code)] // as yet unused, will be used soon /// Derive an xorname from the mnemonic for the account to store data. -pub(super) fn account_root_xorname(mnemonic: bip39::Mnemonic, passphrase: &str) -> Result { +pub(crate) fn account_root_xorname(mnemonic: bip39::Mnemonic, passphrase: &str) -> Result { let seed = mnemonic.to_seed(passphrase); let root_sk = eip2333::derive_master_sk(&seed) @@ -55,13 +56,14 @@ pub(super) fn account_root_xorname(mnemonic: bip39::Mnemonic, passphrase: &str) Ok(XorName::from_content(&derived_key_bytes)) } -pub(super) fn write_mnemonic_to_disk(files_dir: &Path, mnemonic: &bip39::Mnemonic) -> Result<()> { +pub fn write_mnemonic_to_disk(files_dir: &Path, mnemonic: &bip39::Mnemonic) -> Result<()> { let filename = files_dir.join(MNEMONIC_FILENAME); let content = mnemonic.to_string(); std::fs::write(filename, content)?; Ok(()) } +#[allow(dead_code)] // as yet unused, will be used soon pub(super) fn read_mnemonic_from_disk(files_dir: &Path) -> Result { let filename = files_dir.join(MNEMONIC_FILENAME); let content = std::fs::read_to_string(filename) diff --git a/sn_cli/src/bin/main.rs b/sn_cli/src/bin/main.rs index 2aa0309041..22748c35c0 100644 --- a/sn_cli/src/bin/main.rs +++ b/sn_cli/src/bin/main.rs @@ -76,7 +76,7 @@ async fn main() -> Result<()> { let client_data_dir_path = get_client_data_dir_path()?; // Perform actions that do not require us connecting to the network and return early if let SubCmd::Wallet(cmds) = &opt.cmd { - if let WalletCmds::Address + if let WalletCmds::Address { .. } | WalletCmds::Balance { .. } | WalletCmds::Create { .. } | WalletCmds::Sign { .. } = cmds diff --git a/sn_cli/src/bin/subcommands/wallet.rs b/sn_cli/src/bin/subcommands/wallet.rs index a4382b9bb4..67ad0bc9f4 100644 --- a/sn_cli/src/bin/subcommands/wallet.rs +++ b/sn_cli/src/bin/subcommands/wallet.rs @@ -33,13 +33,6 @@ impl WalletApiHelper { Ok(Self::HotWallet(wallet)) } - pub fn address(&self) -> MainPubkey { - match self { - Self::WatchOnlyWallet(w) => w.address(), - Self::HotWallet(w) => w.address(), - } - } - pub fn balance(&self) -> NanoTokens { match self { Self::WatchOnlyWallet(w) => w.balance(), diff --git a/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs b/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs index 0641ade987..0673af0fbb 100644 --- a/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs +++ b/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs @@ -12,6 +12,11 @@ use super::{ WalletApiHelper, }; use crate::get_stdin_response; + +use autonomi::user_secret::{ + account_wallet_secret_key, random_eip2333_mnemonic, write_mnemonic_to_disk, +}; + use bls::SecretKey; use clap::Parser; use color_eyre::{ @@ -31,7 +36,10 @@ use std::{path::Path, str::FromStr}; #[derive(Parser, Debug)] pub enum WalletCmds { /// Print the wallet address. - Address, + Address { + /// Optional passphrase to use for the wallet deriviation if generating a new mnemonic. + passphrase: Option, + }, /// Print the wallet balance. Balance { /// Instead of checking CLI local wallet balance, the PeerId of a node can be used @@ -116,8 +124,31 @@ pub enum WalletCmds { pub(crate) async fn wallet_cmds_without_client(cmds: &WalletCmds, root_dir: &Path) -> Result<()> { match cmds { - WalletCmds::Address => { - let wallet = WalletApiHelper::load_from(root_dir)?; + WalletCmds::Address { passphrase } => { + let wallet = match HotWallet::load_from(root_dir) { + Ok(wallet) => wallet, + Err(error) => match error { + WalletError::MainSecretKeyNotFound(path) => { + warn!("Main secret key not found at {:?}", path); + + let mnemonic = random_eip2333_mnemonic()?; + + write_mnemonic_to_disk(root_dir, &mnemonic)?; + + // TODO grab this as input + let passphrase = passphrase.clone().unwrap_or("default".to_string()); + let sk = account_wallet_secret_key(mnemonic, &passphrase)?; + + // so we generate entropy and create a new wallet + // AccountPacket::get_or_generate_wallet_from_mnemonic(&self, passphrase) + HotWallet::create_from_key(root_dir, sk)? + } + _ => { + return Err(error.into()); + } + }, + }; + println!("{:?}", wallet.address()); Ok(()) } diff --git a/sn_cli/src/lib.rs b/sn_cli/src/lib.rs index 0a85ce69b3..beaa9e1d71 100644 --- a/sn_cli/src/lib.rs +++ b/sn_cli/src/lib.rs @@ -9,6 +9,7 @@ mod acc_packet; mod files; +pub use acc_packet::user_secret; pub use acc_packet::AccountPacket; pub use files::{ download_file, download_files, ChunkManager, Estimator, FilesUploadStatusNotifier, diff --git a/sn_transfers/src/genesis.rs b/sn_transfers/src/genesis.rs index 6b36a03bdc..a6b1682577 100644 --- a/sn_transfers/src/genesis.rs +++ b/sn_transfers/src/genesis.rs @@ -190,7 +190,11 @@ pub fn create_faucet_wallet() -> HotWallet { let root_dir = get_faucet_data_dir(); println!("Loading faucet wallet... {root_dir:#?}"); - HotWallet::load_from(&root_dir).expect("Faucet wallet shall be created successfully.") + + let random_faucet_key = MainSecretKey::random(); + + HotWallet::create_from_key(&root_dir, random_faucet_key) + .expect("Faucet wallet shall be created successfully.") } // We need deterministic and fix path for the faucet wallet. diff --git a/sn_transfers/src/wallet/keys.rs b/sn_transfers/src/wallet/keys.rs index 6006ac44a4..39fc15b767 100644 --- a/sn_transfers/src/wallet/keys.rs +++ b/sn_transfers/src/wallet/keys.rs @@ -83,7 +83,7 @@ mod test { let dir = create_temp_dir(); let root_dir = dir.path().to_path_buf(); store_new_keypair(&root_dir, &main_key)?; - let secret_result = get_main_key_from_disk(&root_dir)?.expect("There to be a key on disk."); + let secret_result = get_main_key_from_disk(&root_dir)?; assert_eq!(secret_result.main_pubkey(), main_key.main_pubkey()); Ok(()) } From a2c9a887c99261df1db44289dc603c9735fa01b5 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Mon, 22 Apr 2024 13:45:56 +0900 Subject: [PATCH 149/205] fix(client): ensure we have a wallet or generate one via mnemonic --- sn_cli/src/bin/subcommands/wallet/helpers.rs | 28 ++++++++++++++- .../src/bin/subcommands/wallet/hot_wallet.rs | 35 ++++--------------- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/sn_cli/src/bin/subcommands/wallet/helpers.rs b/sn_cli/src/bin/subcommands/wallet/helpers.rs index 439da7b82f..474c71ed6b 100644 --- a/sn_cli/src/bin/subcommands/wallet/helpers.rs +++ b/sn_cli/src/bin/subcommands/wallet/helpers.rs @@ -8,6 +8,9 @@ #[cfg(feature = "distribution")] use super::WalletApiHelper; +use autonomi::user_secret::{ + account_wallet_secret_key, random_eip2333_mnemonic, write_mnemonic_to_disk, +}; #[cfg(feature = "distribution")] use base64::Engine; use color_eyre::Result; @@ -16,6 +19,8 @@ use sn_client::Client; use std::{path::Path, str::FromStr}; use url::Url; +const DEFAULT_WALLET_DERIVIATION_PASSPHRASE: &str = "default"; + #[cfg(feature = "distribution")] pub async fn get_faucet( root_dir: &Path, @@ -49,8 +54,29 @@ pub async fn get_faucet( get_faucet_fixed_amount(root_dir, client, url).await } +/// +pub fn load_wallet_or_create_with_mnemonic( + root_dir: &Path, + derivation_passphrase: Option<&str>, +) -> Result { + let wallet = HotWallet::load_from(root_dir); + match wallet { + Ok(wallet) => Ok(wallet), + Err(error) => { + warn!("Issue loading wallet, creating a new one: {error}"); + let mnemonic = random_eip2333_mnemonic()?; + write_mnemonic_to_disk(root_dir, &mnemonic)?; + + let passphrase = derivation_passphrase.unwrap_or(DEFAULT_WALLET_DERIVIATION_PASSPHRASE); + + let wallet = account_wallet_secret_key(mnemonic, passphrase)?; + Ok(HotWallet::create_from_key(root_dir, wallet)?) + } + } +} + pub async fn get_faucet_fixed_amount(root_dir: &Path, client: &Client, url: String) -> Result<()> { - let wallet = HotWallet::load_from(root_dir)?; + let wallet = load_wallet_or_create_with_mnemonic(root_dir, None)?; let address_hex = wallet.address().to_hex(); let url = if !url.contains("://") { format!("{}://{}", "http", url) diff --git a/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs b/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs index 0673af0fbb..606fc124dc 100644 --- a/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs +++ b/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs @@ -8,15 +8,11 @@ use super::{ audit::audit, - helpers::{get_faucet, receive, verify_spend_at}, + helpers::{get_faucet, load_wallet_or_create_with_mnemonic, receive, verify_spend_at}, WalletApiHelper, }; use crate::get_stdin_response; -use autonomi::user_secret::{ - account_wallet_secret_key, random_eip2333_mnemonic, write_mnemonic_to_disk, -}; - use bls::SecretKey; use clap::Parser; use color_eyre::{ @@ -124,30 +120,11 @@ pub enum WalletCmds { pub(crate) async fn wallet_cmds_without_client(cmds: &WalletCmds, root_dir: &Path) -> Result<()> { match cmds { - WalletCmds::Address { passphrase } => { - let wallet = match HotWallet::load_from(root_dir) { - Ok(wallet) => wallet, - Err(error) => match error { - WalletError::MainSecretKeyNotFound(path) => { - warn!("Main secret key not found at {:?}", path); - - let mnemonic = random_eip2333_mnemonic()?; - - write_mnemonic_to_disk(root_dir, &mnemonic)?; - - // TODO grab this as input - let passphrase = passphrase.clone().unwrap_or("default".to_string()); - let sk = account_wallet_secret_key(mnemonic, &passphrase)?; - - // so we generate entropy and create a new wallet - // AccountPacket::get_or_generate_wallet_from_mnemonic(&self, passphrase) - HotWallet::create_from_key(root_dir, sk)? - } - _ => { - return Err(error.into()); - } - }, - }; + WalletCmds::Address { + passphrase: derivation_passphrase, + } => { + let wallet = + load_wallet_or_create_with_mnemonic(root_dir, derivation_passphrase.as_deref())?; println!("{:?}", wallet.address()); Ok(()) From f5f023308f4f972d4533b29b2f922d37ea9bfe9b Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Mon, 22 Apr 2024 13:53:01 +0900 Subject: [PATCH 150/205] fix(client): calm down broadcast error logs if we've no listeners --- sn_client/src/event.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sn_client/src/event.rs b/sn_client/src/event.rs index 25b18eef1c..14ba654d0f 100644 --- a/sn_client/src/event.rs +++ b/sn_client/src/event.rs @@ -29,9 +29,10 @@ impl ClientEventsBroadcaster { // Broadcast a new event, meant to be a helper only used by the client's internals. pub(crate) fn broadcast(&self, event: ClientEvent) { if let Err(err) = self.0.send(event) { - trace!( - "Could not broadcast ClientEvent as we don't have any active listeners: {err:?}" - ); + if self.0.receiver_count() == 0 { + return; + } + trace!("Could not broadcast ClientEvent, though we do have listeners: {err:?}"); } } } From 3e0d6b312b8d6668c224f56975265d2b0dccf25a Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Mon, 22 Apr 2024 14:19:58 +0900 Subject: [PATCH 151/205] fix(client): move acct_packet mnemonic into client layer Allows for reuse in faucet works and proper ci usage. Makes the account packet tooling more client layer there. --- Cargo.lock | 6 +- sn_cli/Cargo.toml | 50 ++++++++++------- sn_cli/src/acc_packet.rs | 1 - sn_cli/src/bin/subcommands/wallet/helpers.rs | 29 +--------- .../src/bin/subcommands/wallet/hot_wallet.rs | 12 ++-- sn_cli/src/lib.rs | 1 - sn_client/Cargo.toml | 37 +++++++++---- sn_client/src/acc_packet.rs | 37 +++++++++++++ .../src/acc_packet/user_secret.rs | 27 ++++----- sn_client/src/error.rs | 12 ++++ sn_client/src/faucet.rs | 47 ++++++++-------- sn_client/src/lib.rs | 3 +- sn_faucet/src/faucet_server.rs | 55 ++++++++++++++----- sn_faucet/src/main.rs | 24 ++++---- sn_transfers/src/wallet/error.rs | 6 +- sn_transfers/src/wallet/hot_wallet.rs | 15 +++-- 16 files changed, 227 insertions(+), 135 deletions(-) create mode 100644 sn_client/src/acc_packet.rs rename {sn_cli => sn_client}/src/acc_packet/user_secret.rs (75%) diff --git a/Cargo.lock b/Cargo.lock index 3f65bd07ad..ff6159cf44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5946,7 +5946,6 @@ version = "0.90.4" dependencies = [ "aes 0.7.5", "base64 0.22.0", - "bip39", "bitcoin", "block-modes", "blsttc", @@ -5968,10 +5967,8 @@ dependencies = [ "reqwest 0.12.4", "rmp-serde", "serde", - "sn_bls_ckd", "sn_build_info", "sn_client", - "sn_curv", "sn_logging", "sn_peers_acquisition", "sn_protocol", @@ -5991,6 +5988,7 @@ dependencies = [ "assert_matches", "async-trait", "backoff", + "bip39", "blsttc", "bytes", "console_error_panic_hook", @@ -6013,7 +6011,9 @@ dependencies = [ "self_encryption", "serde", "serde_json", + "sn_bls_ckd", "sn_client", + "sn_curv", "sn_logging", "sn_networking", "sn_peers_acquisition", diff --git a/sn_cli/Cargo.toml b/sn_cli/Cargo.toml index 10d767dc0f..f4f2c3990d 100644 --- a/sn_cli/Cargo.toml +++ b/sn_cli/Cargo.toml @@ -11,12 +11,12 @@ repository = "https://github.com/maidsafe/safe_network" version = "0.90.4" [[bin]] -path="src/bin/main.rs" -name="safe" +path = "src/bin/main.rs" +name = "safe" [lib] -path="src/lib.rs" -name="autonomi" +path = "src/lib.rs" +name = "autonomi" [[bench]] name = "files" @@ -25,44 +25,54 @@ harness = false [features] default = ["metrics"] distribution = ["base64", "bitcoin"] -local-discovery=["sn_client/local-discovery", "sn_peers_acquisition/local-discovery"] +local-discovery = [ + "sn_client/local-discovery", + "sn_peers_acquisition/local-discovery", +] metrics = ["sn_logging/process-metrics"] network-contacts = ["sn_peers_acquisition/network-contacts"] open-metrics = ["sn_client/open-metrics"] [dependencies] -aes="0.7.5" +aes = "0.7.5" base64 = { version = "0.22.0", optional = true } -bip39 = "2.0.0" -curv = { version = "0.10.1", package = "sn_curv", default-features = false, features = ["num-bigint"] } -bitcoin = { version = "0.31.0", optional = true} -block-modes="0.8.1" +bitcoin = { version = "0.31.0", optional = true } +block-modes = "0.8.1" bls = { package = "blsttc", version = "8.0.1" } bytes = { version = "1.0.1", features = ["serde"] } custom_debug = "~0.6.1" chrono = "~0.4.19" -clap = { version = "4.2.1", features = ["derive"]} +clap = { version = "4.2.1", features = ["derive"] } color-eyre = "~0.6" dialoguer = "~0.11.0" dirs-next = "~2.0.0" -eip2333 = { version = "0.2.1", package = "sn_bls_ckd" } futures = "~0.3.13" hex = "~0.4.3" indicatif = { version = "0.17.5", features = ["tokio"] } -libp2p = { version="0.53", features = ["identify", "kad"] } +libp2p = { version = "0.53", features = ["identify", "kad"] } rand = "0.8.5" rayon = "1.8.0" -reqwest = { version="0.12.2", default-features=false, features = ["rustls-tls-manual-roots"] } +reqwest = { version = "0.12.2", default-features = false, features = [ + "rustls-tls-manual-roots", +] } rmp-serde = "1.1.1" -serde = { version = "1.0.133", features = [ "derive"]} -sn_build_info = { path="../sn_build_info", version = "0.1.7" } +serde = { version = "1.0.133", features = ["derive"] } +sn_build_info = { path = "../sn_build_info", version = "0.1.7" } sn_client = { path = "../sn_client", version = "0.105.3" } sn_logging = { path = "../sn_logging", version = "0.2.25" } -sn_peers_acquisition= { path="../sn_peers_acquisition", version = "0.2.10" } +sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.10" } sn_protocol = { path = "../sn_protocol", version = "0.16.3" } tempfile = "3.6.0" tiny-keccak = "~2.0.2" -tokio = { version = "1.32.0", features = ["io-util", "macros", "parking_lot", "rt", "sync", "time", "fs"] } +tokio = { version = "1.32.0", features = [ + "io-util", + "macros", + "parking_lot", + "rt", + "sync", + "time", + "fs", +] } tracing = { version = "~0.1.26" } url = "2.4.0" walkdir = "~2.5.0" @@ -73,7 +83,9 @@ eyre = "0.6.8" criterion = "0.5.1" tempfile = "3.6.0" rand = { version = "~0.8.5", features = ["small_rng"] } -sn_client = { path = "../sn_client", version = "0.105.3", features = ["test-utils"] } +sn_client = { path = "../sn_client", version = "0.105.3", features = [ + "test-utils", +] } [lints] workspace = true diff --git a/sn_cli/src/acc_packet.rs b/sn_cli/src/acc_packet.rs index d32a12ef21..b1395bb548 100644 --- a/sn_cli/src/acc_packet.rs +++ b/sn_cli/src/acc_packet.rs @@ -7,7 +7,6 @@ // permissions and limitations relating to use of the SAFE Network Software. mod change_tracking; -pub mod user_secret; use change_tracking::*; diff --git a/sn_cli/src/bin/subcommands/wallet/helpers.rs b/sn_cli/src/bin/subcommands/wallet/helpers.rs index 474c71ed6b..682daa781d 100644 --- a/sn_cli/src/bin/subcommands/wallet/helpers.rs +++ b/sn_cli/src/bin/subcommands/wallet/helpers.rs @@ -8,19 +8,15 @@ #[cfg(feature = "distribution")] use super::WalletApiHelper; -use autonomi::user_secret::{ - account_wallet_secret_key, random_eip2333_mnemonic, write_mnemonic_to_disk, -}; #[cfg(feature = "distribution")] use base64::Engine; use color_eyre::Result; +use sn_client::acc_packet::load_account_wallet_or_create_with_mnemonic; use sn_client::transfers::{HotWallet, SpendAddress, Transfer}; use sn_client::Client; use std::{path::Path, str::FromStr}; use url::Url; -const DEFAULT_WALLET_DERIVIATION_PASSPHRASE: &str = "default"; - #[cfg(feature = "distribution")] pub async fn get_faucet( root_dir: &Path, @@ -54,29 +50,8 @@ pub async fn get_faucet( get_faucet_fixed_amount(root_dir, client, url).await } -/// -pub fn load_wallet_or_create_with_mnemonic( - root_dir: &Path, - derivation_passphrase: Option<&str>, -) -> Result { - let wallet = HotWallet::load_from(root_dir); - match wallet { - Ok(wallet) => Ok(wallet), - Err(error) => { - warn!("Issue loading wallet, creating a new one: {error}"); - let mnemonic = random_eip2333_mnemonic()?; - write_mnemonic_to_disk(root_dir, &mnemonic)?; - - let passphrase = derivation_passphrase.unwrap_or(DEFAULT_WALLET_DERIVIATION_PASSPHRASE); - - let wallet = account_wallet_secret_key(mnemonic, passphrase)?; - Ok(HotWallet::create_from_key(root_dir, wallet)?) - } - } -} - pub async fn get_faucet_fixed_amount(root_dir: &Path, client: &Client, url: String) -> Result<()> { - let wallet = load_wallet_or_create_with_mnemonic(root_dir, None)?; + let wallet = load_account_wallet_or_create_with_mnemonic(root_dir, None)?; let address_hex = wallet.address().to_hex(); let url = if !url.contains("://") { format!("{}://{}", "http", url) diff --git a/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs b/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs index 606fc124dc..aa251fe384 100644 --- a/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs +++ b/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs @@ -8,7 +8,7 @@ use super::{ audit::audit, - helpers::{get_faucet, load_wallet_or_create_with_mnemonic, receive, verify_spend_at}, + helpers::{get_faucet, receive, verify_spend_at}, WalletApiHelper, }; use crate::get_stdin_response; @@ -24,7 +24,9 @@ use sn_client::transfers::{ HotWallet, MainPubkey, MainSecretKey, NanoTokens, Transfer, TransferError, UnsignedTransfer, WalletError, }; -use sn_client::{Client, Error as ClientError}; +use sn_client::{ + acc_packet::load_account_wallet_or_create_with_mnemonic, Client, Error as ClientError, +}; use std::{path::Path, str::FromStr}; // Please do not remove the blank lines in these doc comments. @@ -123,8 +125,10 @@ pub(crate) async fn wallet_cmds_without_client(cmds: &WalletCmds, root_dir: &Pat WalletCmds::Address { passphrase: derivation_passphrase, } => { - let wallet = - load_wallet_or_create_with_mnemonic(root_dir, derivation_passphrase.as_deref())?; + let wallet = load_account_wallet_or_create_with_mnemonic( + root_dir, + derivation_passphrase.as_deref(), + )?; println!("{:?}", wallet.address()); Ok(()) diff --git a/sn_cli/src/lib.rs b/sn_cli/src/lib.rs index beaa9e1d71..0a85ce69b3 100644 --- a/sn_cli/src/lib.rs +++ b/sn_cli/src/lib.rs @@ -9,7 +9,6 @@ mod acc_packet; mod files; -pub use acc_packet::user_secret; pub use acc_packet::AccountPacket; pub use files::{ download_file, download_files, ChunkManager, Estimator, FilesUploadStatusNotifier, diff --git a/sn_client/Cargo.toml b/sn_client/Cargo.toml index 3bd0c2e36a..c42b989d37 100644 --- a/sn_client/Cargo.toml +++ b/sn_client/Cargo.toml @@ -11,14 +11,19 @@ repository = "https://github.com/maidsafe/safe_network" version = "0.105.3" [features] -default=[] -local-discovery=["sn_networking/local-discovery"] +default = [] +local-discovery = ["sn_networking/local-discovery"] open-metrics = ["sn_networking/open-metrics", "prometheus-client"] -test-utils=["sn_peers_acquisition", "lazy_static", "eyre"] +test-utils = ["sn_peers_acquisition", "lazy_static", "eyre"] # required to pass on flag to node builds websockets = ["sn_networking/websockets", "sn_protocol/websockets"] [dependencies] +bip39 = "2.0.0" +curv = { version = "0.10.1", package = "sn_curv", default-features = false, features = [ + "num-bigint", +] } +eip2333 = { version = "0.2.1", package = "sn_bls_ckd" } async-trait = "0.1" backoff = { version = "0.4.0", features = ["tokio"] } bls = { package = "blsttc", version = "8.0.1" } @@ -28,14 +33,14 @@ custom_debug = "~0.6.1" futures = "~0.3.13" hex = "~0.4.3" itertools = "~0.12.1" -libp2p = { version="0.53", features = ["identify"] } +libp2p = { version = "0.53", features = ["identify"] } petgraph = { version = "0.6.4", features = ["serde-1"] } prometheus-client = { version = "0.22", optional = true } rand = { version = "~0.8.5", features = ["small_rng"] } rayon = "1.8.0" rmp-serde = "1.1.1" self_encryption = "~0.29.0" -serde = { version = "1.0.133", features = [ "derive", "rc" ]} +serde = { version = "1.0.133", features = ["derive", "rc"] } sn_networking = { path = "../sn_networking", version = "0.14.4" } sn_protocol = { path = "../sn_protocol", version = "0.16.3" } serde_json = "1.0" @@ -44,10 +49,16 @@ sn_transfers = { path = "../sn_transfers", version = "0.17.2" } tempfile = "3.6.0" thiserror = "1.0.23" tiny-keccak = "~2.0.2" -tokio = { version = "1.35.0", features = ["io-util", "macros", "rt", "sync", "time"] } +tokio = { version = "1.35.0", features = [ + "io-util", + "macros", + "rt", + "sync", + "time", +] } tracing = { version = "~0.1.26" } xor_name = "5.0.0" -sn_peers_acquisition= { path="../sn_peers_acquisition", version = "0.2.10", optional = true } +sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.10", optional = true } eyre = { version = "0.6.8", optional = true } lazy_static = { version = "~1.4.0", optional = true } @@ -55,10 +66,12 @@ lazy_static = { version = "~1.4.0", optional = true } assert_matches = "1.5.0" dirs-next = "~2.0.0" # add rand to libp2p -libp2p-identity = { version="0.2.7", features = ["rand"] } -sn_client = { path = "../sn_client", features = ["test-utils"]} +libp2p-identity = { version = "0.2.7", features = ["rand"] } +sn_client = { path = "../sn_client", features = ["test-utils"] } sn_logging = { path = "../sn_logging", version = "0.2.25" } -sn_registers = { path = "../sn_registers", version = "0.3.12", features = ["test-utils"]} +sn_registers = { path = "../sn_registers", version = "0.3.12", features = [ + "test-utils", +] } [lints] workspace = true @@ -71,8 +84,8 @@ crate-type = ["cdylib", "rlib"] getrandom = { version = "0.2.12", features = ["js"] } wasm-bindgen = "0.2.90" wasm-bindgen-futures = "0.4.40" -sn_peers_acquisition= { path="../sn_peers_acquisition", version = "0.2.10" } +sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.10" } console_error_panic_hook = "0.1.6" tracing-wasm = "0.2.1" wasmtimer = "0.2.0" -web-sys = { version= "0.3.22", features=["console"] } +web-sys = { version = "0.3.22", features = ["console"] } diff --git a/sn_client/src/acc_packet.rs b/sn_client/src/acc_packet.rs new file mode 100644 index 0000000000..502bdba14f --- /dev/null +++ b/sn_client/src/acc_packet.rs @@ -0,0 +1,37 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use std::path::Path; + +use super::error::Result; +use sn_transfers::HotWallet; + +pub mod user_secret; + +const DEFAULT_WALLET_DERIVIATION_PASSPHRASE: &str = "default"; + +/// Load a account from disk, with wallet, or create a new one using the mnemonic system +pub fn load_account_wallet_or_create_with_mnemonic( + root_dir: &Path, + derivation_passphrase: Option<&str>, +) -> Result { + let wallet = HotWallet::load_from(root_dir); + match wallet { + Ok(wallet) => Ok(wallet), + Err(error) => { + warn!("Issue loading wallet, creating a new one: {error}"); + let mnemonic = user_secret::random_eip2333_mnemonic()?; + user_secret::write_mnemonic_to_disk(root_dir, &mnemonic)?; + + let passphrase = derivation_passphrase.unwrap_or(DEFAULT_WALLET_DERIVIATION_PASSPHRASE); + + let wallet = user_secret::account_wallet_secret_key(mnemonic, passphrase)?; + Ok(HotWallet::create_from_key(root_dir, wallet)?) + } + } +} diff --git a/sn_cli/src/acc_packet/user_secret.rs b/sn_client/src/acc_packet/user_secret.rs similarity index 75% rename from sn_cli/src/acc_packet/user_secret.rs rename to sn_client/src/acc_packet/user_secret.rs index 50724ba7a6..b866a609a6 100644 --- a/sn_cli/src/acc_packet/user_secret.rs +++ b/sn_client/src/acc_packet/user_secret.rs @@ -6,11 +6,13 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. +use crate::{ + error::{Error, Result}, + transfers::MainSecretKey, +}; use bls::SecretKey; -use color_eyre::{eyre::eyre, Result}; use curv::elliptic::curves::ECScalar; use rand::RngCore; -use sn_client::transfers::MainSecretKey; use std::path::Path; use xor_name::XorName; @@ -24,7 +26,8 @@ pub fn random_eip2333_mnemonic() -> Result { let mut entropy = [1u8; 32]; let rng = &mut rand::rngs::OsRng; rng.fill_bytes(&mut entropy); - let mnemonic = bip39::Mnemonic::from_entropy(&entropy)?; + let mnemonic = + bip39::Mnemonic::from_entropy(&entropy).map_err(|_error| Error::FailedToParseEntropy)?; Ok(mnemonic) } @@ -35,12 +38,11 @@ pub fn account_wallet_secret_key( ) -> Result { let seed = mnemonic.to_seed(passphrase); - let root_sk = eip2333::derive_master_sk(&seed) - .map_err(|err| eyre!("Invalid seed from mnemonic: {err}"))?; + let root_sk = + eip2333::derive_master_sk(&seed).map_err(|_err| Error::InvalidMnemonicSeedPhrase)?; let derived_key = eip2333::derive_child_sk(root_sk, ACCOUNT_WALLET_DERIVATION); let key_bytes = derived_key.serialize(); - let sk = - SecretKey::from_bytes(key_bytes.into()).map_err(|err| eyre!("Invalid key bytes: {err}"))?; + let sk = SecretKey::from_bytes(key_bytes.into()).map_err(|_err| Error::InvalidKeyBytes)?; Ok(MainSecretKey::new(sk)) } @@ -49,8 +51,8 @@ pub fn account_wallet_secret_key( pub(crate) fn account_root_xorname(mnemonic: bip39::Mnemonic, passphrase: &str) -> Result { let seed = mnemonic.to_seed(passphrase); - let root_sk = eip2333::derive_master_sk(&seed) - .map_err(|err| eyre!("Invalid seed from mnemonic: {err}"))?; + let root_sk = + eip2333::derive_master_sk(&seed).map_err(|_err| Error::InvalidMnemonicSeedPhrase)?; let derived_key = eip2333::derive_child_sk(root_sk, ACCOUNT_ROOT_XORNAME_DERIVATION); let derived_key_bytes = derived_key.serialize(); Ok(XorName::from_content(&derived_key_bytes)) @@ -66,9 +68,8 @@ pub fn write_mnemonic_to_disk(files_dir: &Path, mnemonic: &bip39::Mnemonic) -> R #[allow(dead_code)] // as yet unused, will be used soon pub(super) fn read_mnemonic_from_disk(files_dir: &Path) -> Result { let filename = files_dir.join(MNEMONIC_FILENAME); - let content = std::fs::read_to_string(filename) - .map_err(|err| eyre!("Error reading account secret: {err}"))?; - let mnemonic = bip39::Mnemonic::parse_normalized(&content) - .map_err(|err| eyre!("Error parsing account secret: {err}"))?; + let content = std::fs::read_to_string(filename)?; + let mnemonic = + bip39::Mnemonic::parse_normalized(&content).map_err(|_err| Error::FailedToParseMnemonic)?; Ok(mnemonic) } diff --git a/sn_client/src/error.rs b/sn_client/src/error.rs index 13bf844ed1..db064691d9 100644 --- a/sn_client/src/error.rs +++ b/sn_client/src/error.rs @@ -142,4 +142,16 @@ pub enum Error { #[error("Error occurred when access wallet file")] FailedToAccessWallet, + + #[error("Error parsing entropy for mnemonic phrase")] + FailedToParseEntropy, + + #[error("Error parsing mnemonic phrase")] + FailedToParseMnemonic, + + #[error("Invalid mnemonic seed phrase")] + InvalidMnemonicSeedPhrase, + + #[error("SecretKey could not be created from the provided bytes")] + InvalidKeyBytes, } diff --git a/sn_client/src/faucet.rs b/sn_client/src/faucet.rs index 80fe9de7ed..b272a56677 100644 --- a/sn_client/src/faucet.rs +++ b/sn_client/src/faucet.rs @@ -7,40 +7,39 @@ // permissions and limitations relating to use of the SAFE Network Software. use crate::{wallet::send, Client, Result}; -use sn_transfers::{ - create_faucet_wallet, load_genesis_wallet, CashNote, HotWallet, MainPubkey, NanoTokens, -}; +use sn_transfers::{load_genesis_wallet, HotWallet}; -/// Returns a cash_note with the requested number of tokens, for use by E2E test instances. -/// Note this will create a faucet having a Genesis balance -pub async fn get_tokens_from_faucet( - amount: NanoTokens, - to: MainPubkey, - client: &Client, -) -> Result { - send( - load_faucet_wallet_from_genesis_wallet(client).await?, - amount, - to, - client, - // we should not need to wait for this - true, - ) - .await -} +// /// Returns a cash_note with the requested number of tokens, for use by E2E test instances. +// /// Note this will create a faucet having a Genesis balance +// pub async fn get_tokens_from_faucet_from_genesis( +// wallet: HotWallet, +// amount: NanoTokens, +// to: MainPubkey, +// client: &Client, +// ) -> Result { +// let wallet = load_account_wallet_or_create_with_mnemonic(root_dir, None)?; + +// send( +// wallet, amount, to, client, // we should not need to wait for this +// true, +// ) +// .await +// } /// Use the client to load the faucet wallet from the genesis Wallet. /// With all balance transferred from the genesis_wallet to the faucet_wallet. -pub async fn load_faucet_wallet_from_genesis_wallet(client: &Client) -> Result { +pub async fn fund_faucet_from_genesis_wallet( + client: &Client, + faucet_wallet: &mut HotWallet, +) -> Result<()> { println!("Loading faucet..."); info!("Loading faucet..."); - let mut faucet_wallet = create_faucet_wallet(); let faucet_balance = faucet_wallet.balance(); if !faucet_balance.is_zero() { println!("Faucet wallet balance: {faucet_balance}"); debug!("Faucet wallet balance: {faucet_balance}"); - return Ok(faucet_wallet); + return Ok(()); } println!("Loading genesis..."); @@ -75,5 +74,5 @@ pub async fn load_faucet_wallet_from_genesis_wallet(client: &Client) -> Result Result<()> { - claim_genesis(client).await.map_err(|err| { + let root_dir = get_faucet_data_dir(); + let wallet = load_account_wallet_or_create_with_mnemonic(&root_dir, None)?; + claim_genesis(client, wallet).await.map_err(|err| { println!("Faucet Server couldn't start as we failed to claim Genesis"); eprintln!("Faucet Server couldn't start as we failed to claim Genesis"); error!("Faucet Server couldn't start as we failed to claim Genesis"); @@ -125,6 +130,18 @@ async fn respond_to_donate_request( transfer_str: String, semaphore: Arc, ) -> std::result::Result { + let faucet_root = get_faucet_data_dir(); + + let mut wallet = match load_account_wallet_or_create_with_mnemonic(&faucet_root, None) { + Ok(wallet) => wallet, + Err(_error) => { + let mut response = Response::new("Could not load wallet".to_string()); + *response.status_mut() = StatusCode::SERVICE_UNAVAILABLE; + + // Either opening the file or locking it failed, indicating rate limiting should occur + return Ok(response); + } + }; let permit = semaphore.try_acquire(); info!("Got donate request with: {transfer_str}"); @@ -138,16 +155,12 @@ async fn respond_to_donate_request( return Ok(response); } - // load wallet - let mut wallet = match load_faucet_wallet_from_genesis_wallet(&client).await { - Ok(w) => w, - Err(err) => { - eprintln!("Failed to load faucet wallet: {err}"); - error!("Failed to load faucet wallet: {err}"); - let mut response = Response::new(format!("Failed to load faucet wallet: {err}")); - *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; - return Ok(response); - } + if let Err(err) = fund_faucet_from_genesis_wallet(&client, &mut wallet).await { + eprintln!("Failed to load + fund faucet wallet: {err}"); + error!("Failed to load + fund faucet wallet: {err}"); + let mut response = Response::new(format!("Failed to load faucet wallet: {err}")); + *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; + return Ok(response); }; // return key is Transfer is empty @@ -203,6 +216,19 @@ async fn respond_to_gift_request( key: String, semaphore: Arc, ) -> std::result::Result { + let faucet_root = get_faucet_data_dir(); + + let from = match load_account_wallet_or_create_with_mnemonic(&faucet_root, None) { + Ok(wallet) => wallet, + Err(_error) => { + let mut response = Response::new("Could not load wallet".to_string()); + *response.status_mut() = StatusCode::SERVICE_UNAVAILABLE; + + // Either opening the file or locking it failed, indicating rate limiting should occur + return Ok(response); + } + }; + let permit = semaphore.try_acquire(); // some rate limiting @@ -216,7 +242,7 @@ async fn respond_to_gift_request( } const GIFT_AMOUNT_SNT: &str = "1"; - match send_tokens(&client, GIFT_AMOUNT_SNT, &key).await { + match send_tokens(&client, from, GIFT_AMOUNT_SNT, &key).await { Ok(transfer) => { println!("Sent tokens to {key}"); debug!("Sent tokens to {key}"); @@ -234,6 +260,9 @@ async fn startup_server(client: Client) -> Result<()> { // Create a semaphore with a single permit let semaphore = Arc::new(Semaphore::new(1)); + let faucet_root = get_faucet_data_dir(); + let _faucet_wallet = load_account_wallet_or_create_with_mnemonic(&faucet_root, None)?; + #[allow(unused)] let mut balances = HashMap::::new(); #[cfg(feature = "distribution")] diff --git a/sn_faucet/src/main.rs b/sn_faucet/src/main.rs index 639c37270b..7db14f219d 100644 --- a/sn_faucet/src/main.rs +++ b/sn_faucet/src/main.rs @@ -16,12 +16,12 @@ use color_eyre::eyre::{bail, eyre, Result}; use faucet_server::{restart_faucet_server, run_faucet_server}; use indicatif::ProgressBar; use sn_client::{ - get_tokens_from_faucet, load_faucet_wallet_from_genesis_wallet, Client, ClientEvent, - ClientEventsBroadcaster, ClientEventsReceiver, + acc_packet::load_account_wallet_or_create_with_mnemonic, fund_faucet_from_genesis_wallet, send, + Client, ClientEvent, ClientEventsBroadcaster, ClientEventsReceiver, }; use sn_logging::{Level, LogBuilder, LogOutputDest}; use sn_peers_acquisition::{get_peers_from_args, PeersArgs}; -use sn_transfers::{get_faucet_data_dir, MainPubkey, NanoTokens, Transfer}; +use sn_transfers::{get_faucet_data_dir, HotWallet, MainPubkey, NanoTokens, Transfer}; use std::{path::PathBuf, time::Duration}; use tokio::{sync::broadcast::error::RecvError, task::JoinHandle}; use tracing::{debug, error, info}; @@ -81,7 +81,9 @@ async fn main() -> Result<()> { }; handle.await?; - if let Err(err) = faucet_cmds(opt.cmd.clone(), &client).await { + let root_dir = get_faucet_data_dir(); + let funded_faucet = load_account_wallet_or_create_with_mnemonic(&root_dir, None)?; + if let Err(err) = faucet_cmds(opt.cmd.clone(), &client, funded_faucet).await { error!("Failed to run faucet cmd {:?} with err {err:?}", opt.cmd) } @@ -185,13 +187,13 @@ enum SubCmd { RestartServer, } -async fn faucet_cmds(cmds: SubCmd, client: &Client) -> Result<()> { +async fn faucet_cmds(cmds: SubCmd, client: &Client, funded_wallet: HotWallet) -> Result<()> { match cmds { SubCmd::ClaimGenesis => { - claim_genesis(client).await?; + claim_genesis(client, funded_wallet).await?; } SubCmd::Send { amount, to } => { - send_tokens(client, &amount, &to).await?; + send_tokens(client, funded_wallet, &amount, &to).await?; } SubCmd::Server => { // shouldn't return except on error @@ -205,9 +207,9 @@ async fn faucet_cmds(cmds: SubCmd, client: &Client) -> Result<()> { Ok(()) } -async fn claim_genesis(client: &Client) -> Result<()> { +async fn claim_genesis(client: &Client, mut wallet: HotWallet) -> Result<()> { for i in 1..6 { - if let Err(e) = load_faucet_wallet_from_genesis_wallet(client).await { + if let Err(e) = fund_faucet_from_genesis_wallet(client, &mut wallet).await { println!("Failed to claim genesis: {e}"); } else { println!("Genesis claimed!"); @@ -219,7 +221,7 @@ async fn claim_genesis(client: &Client) -> Result<()> { } /// returns the hex-encoded transfer -async fn send_tokens(client: &Client, amount: &str, to: &str) -> Result { +async fn send_tokens(client: &Client, from: HotWallet, amount: &str, to: &str) -> Result { let to = MainPubkey::from_hex(to)?; use std::str::FromStr; let amount = NanoTokens::from_str(amount)?; @@ -230,7 +232,7 @@ async fn send_tokens(client: &Client, amount: &str, to: &str) -> Result )); } - let cash_note = get_tokens_from_faucet(amount, to, client).await?; + let cash_note = send(from, amount, to, client, true).await?; let transfer_hex = Transfer::transfer_from_cash_note(&cash_note)?.to_hex()?; println!("{transfer_hex}"); diff --git a/sn_transfers/src/wallet/error.rs b/sn_transfers/src/wallet/error.rs index 06348e11df..bdbd2a1d49 100644 --- a/sn_transfers/src/wallet/error.rs +++ b/sn_transfers/src/wallet/error.rs @@ -6,7 +6,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::UniquePubkey; +use crate::{MainPubkey, UniquePubkey}; use std::collections::BTreeSet; use thiserror::Error; use xor_name::XorName; @@ -17,6 +17,10 @@ pub type Result = std::result::Result; /// Transfer errors. #[derive(Debug, Error)] pub enum Error { + /// The cashnotes that were attempted to be spent have already been spent to another address + #[error("Attempted to reload a wallet from disk, but the disk wallet is not the same as the current wallet. Current wallet: {0:?}, Disk wallet: {1:?}")] + CurrentAndLoadedKeyMismatch(MainPubkey, MainPubkey), + /// The cashnotes that were attempted to be spent have already been spent to another address #[error("Double spend attempted with cashnotes: {0:?}")] DoubleSpendAttemptedForCashNotes(BTreeSet), diff --git a/sn_transfers/src/wallet/hot_wallet.rs b/sn_transfers/src/wallet/hot_wallet.rs index ddfce397ff..985543f243 100644 --- a/sn_transfers/src/wallet/hot_wallet.rs +++ b/sn_transfers/src/wallet/hot_wallet.rs @@ -63,11 +63,16 @@ impl HotWallet { /// reloads the wallet from disk. fn reload(&mut self) -> Result<()> { // placeholder random MainSecretKey to take it out - let current_key = std::mem::replace(&mut self.key, MainSecretKey::random()); - let wallet = - Self::load_from_path_and_key(self.watchonly_wallet.wallet_dir(), Some(current_key))?; + let wallet = Self::load_from_path_and_key(self.watchonly_wallet.wallet_dir(), None)?; - // and move the original back in + if *wallet.key.secret_key() != *self.key.secret_key() { + return Err(WalletError::CurrentAndLoadedKeyMismatch( + wallet.key.main_pubkey(), + self.key.main_pubkey(), + )); + } + + // if it's a matching key, we can overwrite our wallet *self = wallet; Ok(()) } @@ -624,7 +629,7 @@ impl HotWallet { let watchonly_wallet = WatchOnlyWallet::load_from(wallet_dir, key.main_pubkey())?; Ok(Self { - key, + key: key, watchonly_wallet, unconfirmed_spend_requests, }) From 4640832280c40455c9ad9d052aeca1d42930106a Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 23 Apr 2024 12:52:40 +0900 Subject: [PATCH 152/205] chore(transfers): reduce error size --- sn_transfers/src/wallet/error.rs | 8 ++++---- sn_transfers/src/wallet/hot_wallet.rs | 5 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/sn_transfers/src/wallet/error.rs b/sn_transfers/src/wallet/error.rs index bdbd2a1d49..63f28ca6fd 100644 --- a/sn_transfers/src/wallet/error.rs +++ b/sn_transfers/src/wallet/error.rs @@ -6,8 +6,8 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::{MainPubkey, UniquePubkey}; -use std::collections::BTreeSet; +use crate::UniquePubkey; +use std::{collections::BTreeSet, path::PathBuf}; use thiserror::Error; use xor_name::XorName; @@ -18,8 +18,8 @@ pub type Result = std::result::Result; #[derive(Debug, Error)] pub enum Error { /// The cashnotes that were attempted to be spent have already been spent to another address - #[error("Attempted to reload a wallet from disk, but the disk wallet is not the same as the current wallet. Current wallet: {0:?}, Disk wallet: {1:?}")] - CurrentAndLoadedKeyMismatch(MainPubkey, MainPubkey), + #[error("Attempted to reload a wallet from disk, but the disk wallet is not the same as the current wallet. Wallet path: {0}")] + CurrentAndLoadedKeyMismatch(PathBuf), /// The cashnotes that were attempted to be spent have already been spent to another address #[error("Double spend attempted with cashnotes: {0:?}")] diff --git a/sn_transfers/src/wallet/hot_wallet.rs b/sn_transfers/src/wallet/hot_wallet.rs index 985543f243..4eb8414757 100644 --- a/sn_transfers/src/wallet/hot_wallet.rs +++ b/sn_transfers/src/wallet/hot_wallet.rs @@ -67,8 +67,7 @@ impl HotWallet { if *wallet.key.secret_key() != *self.key.secret_key() { return Err(WalletError::CurrentAndLoadedKeyMismatch( - wallet.key.main_pubkey(), - self.key.main_pubkey(), + self.watchonly_wallet.wallet_dir().into(), )); } @@ -629,7 +628,7 @@ impl HotWallet { let watchonly_wallet = WatchOnlyWallet::load_from(wallet_dir, key.main_pubkey())?; Ok(Self { - key: key, + key, watchonly_wallet, unconfirmed_spend_requests, }) From 5fe8f9b92d527533aab331a27ac4cee133367c6f Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 23 Apr 2024 13:02:22 +0900 Subject: [PATCH 153/205] fix(client): set uploader to use mnemonic wallet loader --- sn_client/src/uploader/upload.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sn_client/src/uploader/upload.rs b/sn_client/src/uploader/upload.rs index 231416e153..f768fa48af 100644 --- a/sn_client/src/uploader/upload.rs +++ b/sn_client/src/uploader/upload.rs @@ -11,6 +11,7 @@ use super::{ UploaderInterface, }; use crate::{ + acc_packet::load_account_wallet_or_create_with_mnemonic, transfers::{TransferError, WalletError}, Client, ClientRegister, Error as ClientError, Result, Uploader, WalletClient, }; @@ -24,7 +25,7 @@ use sn_protocol::{ NetworkAddress, }; use sn_registers::{Register, RegisterAddress}; -use sn_transfers::{HotWallet, NanoTokens, WalletApi}; +use sn_transfers::{NanoTokens, WalletApi}; use std::{ collections::{BTreeMap, BTreeSet, HashMap}, path::{Path, PathBuf}, @@ -797,6 +798,7 @@ impl InnerUploader { batch_size: usize, ) -> Result<()> { let mut wallet_client = Self::load_wallet_client(self.client.clone(), &self.root_dir)?; + let verify_store = self.cfg.verify_store; let _handle = tokio::spawn(async move { debug!("Spawning the long running make payment processing loop."); @@ -1021,8 +1023,7 @@ impl InnerUploader { /// Create a new WalletClient for a given root directory. fn load_wallet_client(client: Client, root_dir: &Path) -> Result { - let wallet = - HotWallet::load_from(root_dir).map_err(|_| ClientError::FailedToAccessWallet)?; + let wallet = load_account_wallet_or_create_with_mnemonic(root_dir, None)?; Ok(WalletClient::new(client, wallet)) } From e8da286dcb2c390ff4db121d2ebd94ba39a8150e Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 23 Apr 2024 13:06:35 +0900 Subject: [PATCH 154/205] feat(cli): readd wallet helper address for dist feat --- sn_cli/src/bin/subcommands/wallet.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sn_cli/src/bin/subcommands/wallet.rs b/sn_cli/src/bin/subcommands/wallet.rs index 67ad0bc9f4..9bca610739 100644 --- a/sn_cli/src/bin/subcommands/wallet.rs +++ b/sn_cli/src/bin/subcommands/wallet.rs @@ -33,6 +33,14 @@ impl WalletApiHelper { Ok(Self::HotWallet(wallet)) } + #[cfg(feature = "distribution")] + pub fn address(&self) -> MainPubkey { + match self { + Self::WatchOnlyWallet(w) => w.address(), + Self::HotWallet(w) => w.address(), + } + } + pub fn balance(&self) -> NanoTokens { match self { Self::WatchOnlyWallet(w) => w.balance(), From b68c7bf17bfa70a62b2ee8df3f9f959b89c90510 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 23 Apr 2024 13:16:07 +0900 Subject: [PATCH 155/205] fix(faucet): fix distribution 'from' wallet loading --- sn_faucet/src/token_distribution.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/sn_faucet/src/token_distribution.rs b/sn_faucet/src/token_distribution.rs index c5e731ef60..7f122ce285 100644 --- a/sn_faucet/src/token_distribution.rs +++ b/sn_faucet/src/token_distribution.rs @@ -11,8 +11,9 @@ use crate::send_tokens; use base64::Engine; use color_eyre::eyre::{eyre, Result}; use serde::{Deserialize, Serialize}; +use sn_client::acc_packet::load_account_wallet_or_create_with_mnemonic; use sn_client::Client; -use sn_transfers::{MainPubkey, NanoTokens}; +use sn_transfers::{get_faucet_data_dir, MainPubkey, NanoTokens}; use std::str::FromStr; use std::{collections::HashMap, path::PathBuf}; use tracing::info; @@ -445,15 +446,19 @@ async fn create_distribution( "Distributing {} for {} to {}", amount, claim.address, claim.wallet ); + + let faucet_dir = get_faucet_data_dir(); + let faucet_wallet = load_account_wallet_or_create_with_mnemonic(&faucet_dir, None)?; // create a transfer to the claim wallet - let transfer_hex = match send_tokens(client, &amount.to_string(), &claim.wallet).await { - Ok(t) => t, - Err(err) => { - let msg = format!("Failed send for {0}: {err}", claim.address); - info!(msg); - return Err(eyre!(msg)); - } - }; + let transfer_hex = + match send_tokens(client, faucet_wallet, &amount.to_string(), &claim.wallet).await { + Ok(t) => t, + Err(err) => { + let msg = format!("Failed send for {0}: {err}", claim.address); + info!(msg); + return Err(eyre!(msg)); + } + }; let _ = match hex::decode(transfer_hex.clone()) { Ok(t) => t, Err(err) => { From f45b92607b975c5b80ea1afb7b3729c32519a51d Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 23 Apr 2024 13:46:22 +0900 Subject: [PATCH 156/205] fix: update calls to HotWallet::load Use account packet helpers to generate wallet reliably now errors out if no wallet is found --- sn_cli/src/acc_packet.rs | 6 ++++-- sn_cli/src/bin/subcommands/register.rs | 6 ++++-- sn_cli/src/bin/subcommands/wallet/audit.rs | 7 +++---- sn_cli/src/bin/subcommands/wallet/helpers.rs | 4 ++-- sn_cli/src/bin/subcommands/wallet/hot_wallet.rs | 6 ++++-- sn_client/src/files.rs | 16 ++++++++-------- sn_client/src/folders.rs | 6 +++--- sn_client/src/test_utils.rs | 6 ++++-- sn_faucet/src/faucet_server.rs | 4 ++-- sn_node/examples/register_inspect.rs | 6 +++--- sn_node/examples/registers.rs | 8 +++++--- 11 files changed, 42 insertions(+), 33 deletions(-) diff --git a/sn_cli/src/acc_packet.rs b/sn_cli/src/acc_packet.rs index b1395bb548..ac9a1a1a5c 100644 --- a/sn_cli/src/acc_packet.rs +++ b/sn_cli/src/acc_packet.rs @@ -683,8 +683,10 @@ impl AccountPacket { let _summary = files_uploader.start_upload().await?; // Let's make the storage payment for Folders - let mut wallet_client = - WalletClient::new(self.client.clone(), HotWallet::load_from(&self.wallet_dir)?); + let new_wallet = MainSecretKey::random(); + let wallet = HotWallet::create_from_key(&self.wallet_dir, new_wallet)?; + + let mut wallet_client = WalletClient::new(self.client.clone(), wallet); let mut net_addresses = vec![]; let mut new_folders = 0; // let's collect list of addresses we need to pay for diff --git a/sn_cli/src/bin/subcommands/register.rs b/sn_cli/src/bin/subcommands/register.rs index 24ece4ae0f..675e1ae6c5 100644 --- a/sn_cli/src/bin/subcommands/register.rs +++ b/sn_cli/src/bin/subcommands/register.rs @@ -9,9 +9,10 @@ use bls::PublicKey; use clap::Subcommand; use color_eyre::{eyre::WrapErr, Result, Section}; +use sn_client::acc_packet::load_account_wallet_or_create_with_mnemonic; use sn_client::protocol::storage::RegisterAddress; use sn_client::registers::Permissions; -use sn_client::transfers::HotWallet; + use sn_client::{Client, Error as ClientError, WalletClient}; use std::path::Path; use xor_name::XorName; @@ -86,7 +87,8 @@ async fn create_register( verify_store: bool, ) -> Result<()> { trace!("Starting to pay for Register storage"); - let wallet = HotWallet::load_from(root_dir) + + let wallet = load_account_wallet_or_create_with_mnemonic(root_dir, None) .wrap_err("Unable to read wallet file in {path:?}") .suggestion( "If you have an old wallet file, it may no longer be compatible. Try removing it", diff --git a/sn_cli/src/bin/subcommands/wallet/audit.rs b/sn_cli/src/bin/subcommands/wallet/audit.rs index 53a60041c6..ca6c8adb15 100644 --- a/sn_cli/src/bin/subcommands/wallet/audit.rs +++ b/sn_cli/src/bin/subcommands/wallet/audit.rs @@ -9,9 +9,8 @@ use std::path::Path; use color_eyre::Result; -use sn_client::transfers::{ - CashNoteRedemption, HotWallet, SpendAddress, Transfer, GENESIS_CASHNOTE, -}; +use sn_client::acc_packet::load_account_wallet_or_create_with_mnemonic; +use sn_client::transfers::{CashNoteRedemption, SpendAddress, Transfer, GENESIS_CASHNOTE}; use sn_client::{Client, SpendDag}; const SPEND_DAG_FILENAME: &str = "spend_dag"; @@ -69,7 +68,7 @@ async fn redeem_royalties( println!("Found {} royalties.", royalties.len()); } - let mut wallet = HotWallet::load_from(root_dir)?; + let mut wallet = load_account_wallet_or_create_with_mnemonic(root_dir, None)?; // batch royalties per 100 let mut batch = Vec::new(); diff --git a/sn_cli/src/bin/subcommands/wallet/helpers.rs b/sn_cli/src/bin/subcommands/wallet/helpers.rs index 682daa781d..f1327f821e 100644 --- a/sn_cli/src/bin/subcommands/wallet/helpers.rs +++ b/sn_cli/src/bin/subcommands/wallet/helpers.rs @@ -12,7 +12,7 @@ use super::WalletApiHelper; use base64::Engine; use color_eyre::Result; use sn_client::acc_packet::load_account_wallet_or_create_with_mnemonic; -use sn_client::transfers::{HotWallet, SpendAddress, Transfer}; +use sn_client::transfers::{SpendAddress, Transfer}; use sn_client::Client; use std::{path::Path, str::FromStr}; use url::Url; @@ -134,7 +134,7 @@ pub async fn receive( println!("Successfully parsed transfer. "); println!("Verifying transfer with the Network..."); - let mut wallet = HotWallet::load_from(root_dir)?; + let mut wallet = load_account_wallet_or_create_with_mnemonic(root_dir, None)?; let cashnotes = match client.receive(&transfer, &wallet).await { Ok(cashnotes) => cashnotes, Err(err) => { diff --git a/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs b/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs index aa251fe384..a73eef0bfc 100644 --- a/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs +++ b/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs @@ -220,7 +220,8 @@ async fn send( root_dir: &Path, verify_store: bool, ) -> Result<()> { - let from = HotWallet::load_from(root_dir)?; + let from = load_account_wallet_or_create_with_mnemonic(root_dir, None)?; + let amount = match NanoTokens::from_str(&amount) { Ok(amount) => amount, Err(err) => { @@ -271,7 +272,8 @@ async fn send( } fn sign_transaction(tx: &str, root_dir: &Path, force: bool) -> Result<()> { - let wallet = HotWallet::load_from(root_dir)?; + let wallet = load_account_wallet_or_create_with_mnemonic(root_dir, None)?; + let unsigned_transfer: UnsignedTransfer = rmp_serde::from_slice(&hex::decode(tx)?)?; println!("The unsigned transaction has been successfully decoded:"); diff --git a/sn_client/src/files.rs b/sn_client/src/files.rs index 5ab6a6252e..36d743d319 100644 --- a/sn_client/src/files.rs +++ b/sn_client/src/files.rs @@ -9,8 +9,8 @@ pub(crate) mod download; use crate::{ - chunks::Error as ChunksError, error::Result, wallet::StoragePaymentResult, Client, Error, - WalletClient, + acc_packet::load_account_wallet_or_create_with_mnemonic, chunks::Error as ChunksError, + error::Result, wallet::StoragePaymentResult, Client, Error, WalletClient, }; use bytes::Bytes; use self_encryption::{self, MIN_ENCRYPTABLE_BYTES}; @@ -18,7 +18,7 @@ use sn_protocol::{ storage::{Chunk, ChunkAddress, RetryStrategy}, NetworkAddress, }; -use sn_transfers::HotWallet; + use std::{ fs::{self, create_dir_all, File}, io::Write, @@ -48,10 +48,9 @@ impl FilesApi { Self { client, wallet_dir } } pub fn build(client: Client, wallet_dir: PathBuf) -> Result { - if HotWallet::load_from(wallet_dir.as_path())? - .balance() - .is_zero() - { + let wallet = load_account_wallet_or_create_with_mnemonic(&wallet_dir, None)?; + + if wallet.balance().is_zero() { Err(Error::AmountIsZero) } else { Ok(FilesApi::new(client, wallet_dir)) @@ -66,7 +65,8 @@ impl FilesApi { /// Create a new WalletClient for a given root directory. pub fn wallet(&self) -> Result { let path = self.wallet_dir.as_path(); - let wallet = HotWallet::load_from(path)?; + + let wallet = load_account_wallet_or_create_with_mnemonic(path, None)?; Ok(WalletClient::new(self.client.clone(), wallet)) } diff --git a/sn_client/src/folders.rs b/sn_client/src/folders.rs index 624a9e7f4e..2993d68ba5 100644 --- a/sn_client/src/folders.rs +++ b/sn_client/src/folders.rs @@ -7,7 +7,7 @@ // permissions and limitations relating to use of the SAFE Network Software. use super::{error::Result, Client, ClientRegister, WalletClient}; -use crate::{Error, FilesApi, UploadCfg}; +use crate::{acc_packet::load_account_wallet_or_create_with_mnemonic, Error, FilesApi, UploadCfg}; use bls::{Ciphertext, PublicKey}; use bytes::{BufMut, BytesMut}; use self_encryption::MAX_CHUNK_SIZE; @@ -17,7 +17,7 @@ use sn_protocol::{ NetworkAddress, }; use sn_registers::{Entry, EntryHash}; -use sn_transfers::HotWallet; + use std::{ collections::{BTreeMap, BTreeSet}, ffi::OsString, @@ -110,7 +110,7 @@ impl FoldersApi { /// Create a new WalletClient from the directory set. pub fn wallet(&self) -> Result { - let wallet = HotWallet::load_from(&self.wallet_dir)?; + let wallet = load_account_wallet_or_create_with_mnemonic(&self.wallet_dir, None)?; Ok(WalletClient::new(self.client.clone(), wallet)) } diff --git a/sn_client/src/test_utils.rs b/sn_client/src/test_utils.rs index bff4b42636..f772a76f89 100644 --- a/sn_client/src/test_utils.rs +++ b/sn_client/src/test_utils.rs @@ -6,7 +6,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::{send, Client, WalletClient}; +use crate::{acc_packet::load_account_wallet_or_create_with_mnemonic, send, Client, WalletClient}; use sn_peers_acquisition::parse_peer_addr; use sn_protocol::{storage::Chunk, NetworkAddress}; use sn_transfers::{create_faucet_wallet, HotWallet, NanoTokens}; @@ -98,7 +98,9 @@ pub async fn pay_for_storage( wallet_dir: &Path, addrs2pay: Vec, ) -> Result<()> { - let mut wallet_client = WalletClient::new(client.clone(), HotWallet::load_from(wallet_dir)?); + let wallet = load_account_wallet_or_create_with_mnemonic(wallet_dir, None)?; + + let mut wallet_client = WalletClient::new(client.clone(), wallet); let _ = wallet_client.pay_for_storage(addrs2pay.into_iter()).await?; Ok(()) } diff --git a/sn_faucet/src/faucet_server.rs b/sn_faucet/src/faucet_server.rs index 4f245875e3..f398e13e53 100644 --- a/sn_faucet/src/faucet_server.rs +++ b/sn_faucet/src/faucet_server.rs @@ -16,7 +16,7 @@ use sn_client::{ Client, }; use sn_transfers::{ - get_faucet_data_dir, wallet_lockfile_name, HotWallet, NanoTokens, Transfer, WALLET_DIR_NAME, + get_faucet_data_dir, wallet_lockfile_name, NanoTokens, Transfer, WALLET_DIR_NAME, }; use std::path::Path; use std::{collections::HashMap, sync::Arc}; @@ -363,7 +363,7 @@ async fn startup_server(client: Client) -> Result<()> { } fn deposit(root_dir: &Path) -> Result<()> { - let mut wallet = HotWallet::load_from(root_dir)?; + let mut wallet = load_account_wallet_or_create_with_mnemonic(root_dir, None)?; let previous_balance = wallet.balance(); diff --git a/sn_node/examples/register_inspect.rs b/sn_node/examples/register_inspect.rs index b404510625..3c3d70a36b 100644 --- a/sn_node/examples/register_inspect.rs +++ b/sn_node/examples/register_inspect.rs @@ -10,9 +10,9 @@ use crdts::merkle_reg::{Hash, MerkleReg, Node}; use std::collections::HashMap; use std::io; -use sn_client::{Client, WalletClient}; +use sn_client::{acc_packet::load_account_wallet_or_create_with_mnemonic, Client, WalletClient}; use sn_registers::{Entry, Permissions, RegisterAddress}; -use sn_transfers::HotWallet; + use xor_name::XorName; use bls::SecretKey; @@ -71,7 +71,7 @@ async fn main() -> Result<()> { .join("safe") .join("client"); - let wallet = HotWallet::load_from(&root_dir) + let wallet = load_account_wallet_or_create_with_mnemonic(&root_dir, None) .wrap_err("Unable to read wallet file in {root_dir:?}") .suggestion( "If you have an old wallet file, it may no longer be compatible. Try removing it", diff --git a/sn_node/examples/registers.rs b/sn_node/examples/registers.rs index cb7b988ae1..70d3177a1c 100644 --- a/sn_node/examples/registers.rs +++ b/sn_node/examples/registers.rs @@ -6,9 +6,11 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use sn_client::{Client, Error, WalletClient}; +use sn_client::{ + acc_packet::load_account_wallet_or_create_with_mnemonic, Client, Error, WalletClient, +}; use sn_registers::{Permissions, RegisterAddress}; -use sn_transfers::HotWallet; + use xor_name::XorName; use bls::SecretKey; @@ -75,7 +77,7 @@ async fn main() -> Result<()> { .join("safe") .join("client"); - let wallet = HotWallet::load_from(&root_dir) + let wallet = load_account_wallet_or_create_with_mnemonic(&root_dir, None) .wrap_err("Unable to read wallet file in {root_dir:?}") .suggestion( "If you have an old wallet file, it may no longer be compatible. Try removing it", From 54952b1a627bee12559065d90b5ea3fb5ee042c2 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 23 Apr 2024 14:07:19 +0900 Subject: [PATCH 157/205] fix: transfer tests for HotWallet creation --- sn_transfers/src/wallet/hot_wallet.rs | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/sn_transfers/src/wallet/hot_wallet.rs b/sn_transfers/src/wallet/hot_wallet.rs index 4eb8414757..2f60d334b6 100644 --- a/sn_transfers/src/wallet/hot_wallet.rs +++ b/sn_transfers/src/wallet/hot_wallet.rs @@ -801,7 +801,8 @@ mod tests { let dir = create_temp_dir(); let root_dir = dir.path().to_path_buf(); - let mut depositor = HotWallet::load_from(&root_dir)?; + let new_wallet = MainSecretKey::random(); + let mut depositor = HotWallet::create_from_key(&root_dir, new_wallet)?; let genesis = create_first_cash_note_from_key(&depositor.key).expect("Genesis creation to succeed."); depositor.deposit_and_store_to_disk(&vec![genesis])?; @@ -844,8 +845,8 @@ mod tests { async fn sending_decreases_balance() -> Result<()> { let dir = create_temp_dir(); let root_dir = dir.path().to_path_buf(); - - let mut sender = HotWallet::load_from(&root_dir)?; + let new_wallet = MainSecretKey::random(); + let mut sender = HotWallet::create_from_key(&root_dir, new_wallet)?; let sender_cash_note = create_first_cash_note_from_key(&sender.key).expect("Genesis creation to succeed."); sender.deposit_and_store_to_disk(&vec![sender_cash_note])?; @@ -877,7 +878,9 @@ mod tests { let dir = create_temp_dir(); let root_dir = dir.path().to_path_buf(); - let mut sender = HotWallet::load_from(&root_dir)?; + let new_wallet = MainSecretKey::random(); + let mut sender = HotWallet::create_from_key(&root_dir, new_wallet)?; + let sender_cash_note = create_first_cash_note_from_key(&sender.key).expect("Genesis creation to succeed."); sender.deposit_and_store_to_disk(&vec![sender_cash_note])?; @@ -929,8 +932,9 @@ mod tests { async fn store_created_cash_note_gives_file_that_try_load_cash_notes_can_use() -> Result<()> { let sender_root_dir = create_temp_dir(); let sender_root_dir = sender_root_dir.path().to_path_buf(); + let new_wallet = MainSecretKey::random(); + let mut sender = HotWallet::create_from_key(&sender_root_dir, new_wallet)?; - let mut sender = HotWallet::load_from(&sender_root_dir)?; let sender_cash_note = create_first_cash_note_from_key(&sender.key).expect("Genesis creation to succeed."); sender.deposit_and_store_to_disk(&vec![sender_cash_note])?; @@ -940,7 +944,10 @@ mod tests { // Send to a new address. let recipient_root_dir = create_temp_dir(); let recipient_root_dir = recipient_root_dir.path().to_path_buf(); - let mut recipient = HotWallet::load_from(&recipient_root_dir)?; + + let new_wallet = MainSecretKey::random(); + let mut recipient = HotWallet::create_from_key(&recipient_root_dir, new_wallet)?; + let recipient_main_pubkey = recipient.key.main_pubkey(); let to = vec![(NanoTokens::from(send_amount), recipient_main_pubkey)]; @@ -987,7 +994,9 @@ mod tests { let dir = create_temp_dir(); let root_dir = dir.path().to_path_buf(); - let mut sender = HotWallet::load_from(&root_dir)?; + let new_wallet = MainSecretKey::random(); + let mut sender = HotWallet::create_from_key(&root_dir, new_wallet)?; + let sender_cash_note = create_first_cash_note_from_key(&sender.key).expect("Genesis creation to succeed."); sender.deposit_and_store_to_disk(&vec![sender_cash_note])?; From 86112a196d9ff357271d34388622bda1b96ef0f8 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 23 Apr 2024 14:11:51 +0900 Subject: [PATCH 158/205] fix: more test and cli fixes --- sn_cli/src/bin/subcommands/wallet.rs | 8 -------- sn_cli/src/bin/subcommands/wallet/helpers.rs | 6 +++--- sn_client/src/test_utils.rs | 5 +++-- sn_node/tests/common/client.rs | 5 +++-- 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/sn_cli/src/bin/subcommands/wallet.rs b/sn_cli/src/bin/subcommands/wallet.rs index 9bca610739..67ad0bc9f4 100644 --- a/sn_cli/src/bin/subcommands/wallet.rs +++ b/sn_cli/src/bin/subcommands/wallet.rs @@ -33,14 +33,6 @@ impl WalletApiHelper { Ok(Self::HotWallet(wallet)) } - #[cfg(feature = "distribution")] - pub fn address(&self) -> MainPubkey { - match self { - Self::WatchOnlyWallet(w) => w.address(), - Self::HotWallet(w) => w.address(), - } - } - pub fn balance(&self) -> NanoTokens { match self { Self::WatchOnlyWallet(w) => w.balance(), diff --git a/sn_cli/src/bin/subcommands/wallet/helpers.rs b/sn_cli/src/bin/subcommands/wallet/helpers.rs index f1327f821e..dee3089a4b 100644 --- a/sn_cli/src/bin/subcommands/wallet/helpers.rs +++ b/sn_cli/src/bin/subcommands/wallet/helpers.rs @@ -6,8 +6,6 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -#[cfg(feature = "distribution")] -use super::WalletApiHelper; #[cfg(feature = "distribution")] use base64::Engine; use color_eyre::Result; @@ -88,7 +86,9 @@ pub async fn get_faucet_distribution( url }; // receive to the current local wallet - let wallet = WalletApiHelper::load_from(root_dir)?.address().to_hex(); + let wallet = load_account_wallet_or_create_with_mnemonic(root_dir, None)? + .address() + .to_hex(); println!("Requesting distribution for maid address {address} to local wallet {wallet}"); // base64 uses + and / as the delimiters which doesn't go well in the query // string, so the signature is encoded using url safe characters. diff --git a/sn_client/src/test_utils.rs b/sn_client/src/test_utils.rs index f772a76f89..edcb8af212 100644 --- a/sn_client/src/test_utils.rs +++ b/sn_client/src/test_utils.rs @@ -67,8 +67,9 @@ pub async fn get_funded_wallet(client: &Client, wallet_dir: &Path) -> Result HotWallet { - HotWallet::load_from(root_dir).expect("Wallet shall be successfully created.") + load_account_wallet_or_create_with_mnemonic(root_dir, None) + .expect("Wallet shall be successfully created.") } /// Get the node count From de68506858abd1f56dbb6e235b38f5f21b106b6a Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 23 Apr 2024 14:29:26 +0900 Subject: [PATCH 159/205] fix: create faucet via account load or generation --- sn_client/src/acc_packet.rs | 10 +++++++++- sn_client/src/test_utils.rs | 9 ++++++--- sn_node/tests/common/client.rs | 9 ++++++--- sn_transfers/src/genesis.rs | 11 ----------- sn_transfers/src/lib.rs | 7 +++---- 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/sn_client/src/acc_packet.rs b/sn_client/src/acc_packet.rs index 502bdba14f..5636e759d1 100644 --- a/sn_client/src/acc_packet.rs +++ b/sn_client/src/acc_packet.rs @@ -9,7 +9,7 @@ use std::path::Path; use super::error::Result; -use sn_transfers::HotWallet; +use sn_transfers::{get_faucet_data_dir, HotWallet}; pub mod user_secret; @@ -35,3 +35,11 @@ pub fn load_account_wallet_or_create_with_mnemonic( } } } + +pub fn create_faucet_account_and_wallet() -> HotWallet { + let root_dir = get_faucet_data_dir(); + + println!("Loading faucet wallet... {root_dir:#?}"); + load_account_wallet_or_create_with_mnemonic(&root_dir, None) + .expect("Faucet wallet shall be created successfully.") +} diff --git a/sn_client/src/test_utils.rs b/sn_client/src/test_utils.rs index edcb8af212..0c0ef2fd0a 100644 --- a/sn_client/src/test_utils.rs +++ b/sn_client/src/test_utils.rs @@ -6,10 +6,13 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::{acc_packet::load_account_wallet_or_create_with_mnemonic, send, Client, WalletClient}; +use crate::{ + acc_packet::{create_faucet_account_and_wallet, load_account_wallet_or_create_with_mnemonic}, + send, Client, WalletClient, +}; use sn_peers_acquisition::parse_peer_addr; use sn_protocol::{storage::Chunk, NetworkAddress}; -use sn_transfers::{create_faucet_wallet, HotWallet, NanoTokens}; +use sn_transfers::{HotWallet, NanoTokens}; use bls::SecretKey; use bytes::Bytes; @@ -110,7 +113,7 @@ async fn load_faucet_wallet() -> Result { info!("Loading faucet..."); let now = Instant::now(); for attempt in 1..LOAD_FAUCET_WALLET_RETRIES + 1 { - let faucet_wallet = create_faucet_wallet(); + let faucet_wallet = create_faucet_account_and_wallet(); let faucet_balance = faucet_wallet.balance(); if !faucet_balance.is_zero() { diff --git a/sn_node/tests/common/client.rs b/sn_node/tests/common/client.rs index 19e8cf83d4..e00ff36058 100644 --- a/sn_node/tests/common/client.rs +++ b/sn_node/tests/common/client.rs @@ -9,13 +9,16 @@ use eyre::{bail, OptionExt, Result}; use lazy_static::lazy_static; use libp2p::PeerId; -use sn_client::{acc_packet::load_account_wallet_or_create_with_mnemonic, send, Client}; +use sn_client::{ + acc_packet::{create_faucet_account_and_wallet, load_account_wallet_or_create_with_mnemonic}, + send, Client, +}; use sn_peers_acquisition::parse_peer_addr; use sn_protocol::safenode_proto::{NodeInfoRequest, RestartRequest}; use sn_service_management::{ get_local_node_registry_path, safenode_manager_proto::NodeServiceRestartRequest, NodeRegistry, }; -use sn_transfers::{create_faucet_wallet, HotWallet, NanoTokens, Transfer}; +use sn_transfers::{HotWallet, NanoTokens, Transfer}; use std::{net::SocketAddr, path::Path}; use test_utils::testnet::DeploymentInventory; use tokio::{ @@ -213,7 +216,7 @@ impl NonDroplet { info!("Loading faucet..."); let now = Instant::now(); for attempt in 1..LOAD_FAUCET_WALLET_RETRIES + 1 { - let faucet_wallet = create_faucet_wallet(); + let faucet_wallet = create_faucet_account_and_wallet(); let faucet_balance = faucet_wallet.balance(); if !faucet_balance.is_zero() { diff --git a/sn_transfers/src/genesis.rs b/sn_transfers/src/genesis.rs index a6b1682577..4d590957dd 100644 --- a/sn_transfers/src/genesis.rs +++ b/sn_transfers/src/genesis.rs @@ -186,17 +186,6 @@ pub fn create_first_cash_note_from_key( Ok(genesis_cash_note) } -pub fn create_faucet_wallet() -> HotWallet { - let root_dir = get_faucet_data_dir(); - - println!("Loading faucet wallet... {root_dir:#?}"); - - let random_faucet_key = MainSecretKey::random(); - - HotWallet::create_from_key(&root_dir, random_faucet_key) - .expect("Faucet wallet shall be created successfully.") -} - // We need deterministic and fix path for the faucet wallet. // Otherwise the test instances will not be able to find the same faucet instance. pub fn get_faucet_data_dir() -> PathBuf { diff --git a/sn_transfers/src/lib.rs b/sn_transfers/src/lib.rs index 58ee385dc0..449247fc70 100644 --- a/sn_transfers/src/lib.rs +++ b/sn_transfers/src/lib.rs @@ -27,10 +27,9 @@ pub use transfers::{CashNoteRedemption, OfflineTransfer, Transfer}; /// Utilities exposed pub use genesis::{ - calculate_royalties_fee, create_faucet_wallet, create_first_cash_note_from_key, - get_faucet_data_dir, is_genesis_parent_tx, is_genesis_spend, load_genesis_wallet, - Error as GenesisError, GENESIS_CASHNOTE, GENESIS_CASHNOTE_SK, NETWORK_ROYALTIES_PK, - TOTAL_SUPPLY, + calculate_royalties_fee, create_first_cash_note_from_key, get_faucet_data_dir, + is_genesis_parent_tx, is_genesis_spend, load_genesis_wallet, Error as GenesisError, + GENESIS_CASHNOTE, GENESIS_CASHNOTE_SK, NETWORK_ROYALTIES_PK, TOTAL_SUPPLY, }; pub use wallet::{ bls_secret_from_hex, wallet_lockfile_name, Error as WalletError, HotWallet, Payment, From 83b45af08675feeec4595cc36d3e302bc6719278 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 23 Apr 2024 17:03:59 +0900 Subject: [PATCH 160/205] fix(cli): acct_packet tests updated --- sn_cli/src/acc_packet.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sn_cli/src/acc_packet.rs b/sn_cli/src/acc_packet.rs index ac9a1a1a5c..26db181fdf 100644 --- a/sn_cli/src/acc_packet.rs +++ b/sn_cli/src/acc_packet.rs @@ -16,9 +16,10 @@ use super::{ }; use sn_client::{ + acc_packet::load_account_wallet_or_create_with_mnemonic, protocol::storage::{Chunk, RegisterAddress, RetryStrategy}, registers::EntryHash, - transfers::{DerivationIndex, HotWallet, MainSecretKey}, + transfers::{DerivationIndex, MainSecretKey}, Client, FilesApi, FolderEntry, FoldersApi, Metadata, UploadCfg, WalletClient, }; @@ -683,8 +684,7 @@ impl AccountPacket { let _summary = files_uploader.start_upload().await?; // Let's make the storage payment for Folders - let new_wallet = MainSecretKey::random(); - let wallet = HotWallet::create_from_key(&self.wallet_dir, new_wallet)?; + let wallet = load_account_wallet_or_create_with_mnemonic(&self.wallet_dir, None)?; let mut wallet_client = WalletClient::new(self.client.clone(), wallet); let mut net_addresses = vec![]; From 2b77798ffa6ed3bad47f3ee9076453cc5110b685 Mon Sep 17 00:00:00 2001 From: qima Date: Tue, 23 Apr 2024 22:22:39 +0800 Subject: [PATCH 161/205] chore(faucet): log initilization failure and upload faucet log --- .github/workflows/memcheck.yml | 17 +++++++++++++++++ sn_faucet/src/main.rs | 9 ++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/memcheck.yml b/.github/workflows/memcheck.yml index 4403dc57e8..3275501204 100644 --- a/.github/workflows/memcheck.yml +++ b/.github/workflows/memcheck.yml @@ -412,6 +412,23 @@ jobs: continue-on-error: true if: always() + - name: Move faucet log to the working folder + run: | + echo "current folder is:" + pwd + echo "SAFE_DATA_PATH has: " + ls -l $SAFE_DATA_PATH + echo "test_faucet foder has: " + ls -l $SAFE_DATA_PATH/test_faucet + echo "logs folder has: " + ls -l $SAFE_DATA_PATH/test_faucet/logs + mv $FAUCET_LOG_PATH/*.log ./faucet_log.log + env: + SN_LOG: "all" + continue-on-error: true + if: always() + timeout-minutes: 1 + - name: Upload faucet log uses: actions/upload-artifact@main with: diff --git a/sn_faucet/src/main.rs b/sn_faucet/src/main.rs index 7db14f219d..2e8e480498 100644 --- a/sn_faucet/src/main.rs +++ b/sn_faucet/src/main.rs @@ -82,7 +82,14 @@ async fn main() -> Result<()> { handle.await?; let root_dir = get_faucet_data_dir(); - let funded_faucet = load_account_wallet_or_create_with_mnemonic(&root_dir, None)?; + let funded_faucet = match load_account_wallet_or_create_with_mnemonic(&root_dir, None) { + Ok(wallet) => wallet, + Err(err) => { + println!("failed to load wallet for faucet! with error {err:?}"); + error!("failed to load wallet for faucet! with error {err:?}"); + return Err(err.into()); + } + }; if let Err(err) = faucet_cmds(opt.cmd.clone(), &client, funded_faucet).await { error!("Failed to run faucet cmd {:?} with err {err:?}", opt.cmd) } From 69c6f64d3f001b93775fa4e4456afe5a0e65d5bf Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Wed, 24 Apr 2024 09:58:58 +0900 Subject: [PATCH 162/205] fix(faucet): ensure faucet is funded in main fn --- sn_faucet/src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sn_faucet/src/main.rs b/sn_faucet/src/main.rs index 2e8e480498..e9d96b529f 100644 --- a/sn_faucet/src/main.rs +++ b/sn_faucet/src/main.rs @@ -82,7 +82,7 @@ async fn main() -> Result<()> { handle.await?; let root_dir = get_faucet_data_dir(); - let funded_faucet = match load_account_wallet_or_create_with_mnemonic(&root_dir, None) { + let mut funded_faucet = match load_account_wallet_or_create_with_mnemonic(&root_dir, None) { Ok(wallet) => wallet, Err(err) => { println!("failed to load wallet for faucet! with error {err:?}"); @@ -90,6 +90,9 @@ async fn main() -> Result<()> { return Err(err.into()); } }; + + fund_faucet_from_genesis_wallet(&client, &mut funded_faucet).await?; + if let Err(err) = faucet_cmds(opt.cmd.clone(), &client, funded_faucet).await { error!("Failed to run faucet cmd {:?} with err {err:?}", opt.cmd) } From 6881d90ffb073e533eab85a6af08744a766394ba Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Wed, 24 Apr 2024 11:06:10 +0900 Subject: [PATCH 163/205] chore: small cleanup of dead code --- sn_client/src/faucet.rs | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/sn_client/src/faucet.rs b/sn_client/src/faucet.rs index b272a56677..415cd88d9f 100644 --- a/sn_client/src/faucet.rs +++ b/sn_client/src/faucet.rs @@ -9,23 +9,6 @@ use crate::{wallet::send, Client, Result}; use sn_transfers::{load_genesis_wallet, HotWallet}; -// /// Returns a cash_note with the requested number of tokens, for use by E2E test instances. -// /// Note this will create a faucet having a Genesis balance -// pub async fn get_tokens_from_faucet_from_genesis( -// wallet: HotWallet, -// amount: NanoTokens, -// to: MainPubkey, -// client: &Client, -// ) -> Result { -// let wallet = load_account_wallet_or_create_with_mnemonic(root_dir, None)?; - -// send( -// wallet, amount, to, client, // we should not need to wait for this -// true, -// ) -// .await -// } - /// Use the client to load the faucet wallet from the genesis Wallet. /// With all balance transferred from the genesis_wallet to the faucet_wallet. pub async fn fund_faucet_from_genesis_wallet( From 619ba2e0a1d3ab3c78ccd6d84f3d9f4fdd2a7af8 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 25 Apr 2024 14:46:39 +0900 Subject: [PATCH 164/205] chore(cli): update mnemonic wallet seed phrase wording --- sn_cli/src/bin/subcommands/wallet/hot_wallet.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs b/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs index a73eef0bfc..e04722c238 100644 --- a/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs +++ b/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs @@ -35,7 +35,7 @@ use std::{path::Path, str::FromStr}; pub enum WalletCmds { /// Print the wallet address. Address { - /// Optional passphrase to use for the wallet deriviation if generating a new mnemonic. + /// Optional phrase to use for the wallet deriviation from mnemonic entropy. passphrase: Option, }, /// Print the wallet balance. From 8df5072a74b8046288fcf0339edfcd89909ab6c8 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 25 Apr 2024 15:36:00 +0900 Subject: [PATCH 165/205] fix(faucet): rate limit before getting wallet --- sn_faucet/src/faucet_server.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sn_faucet/src/faucet_server.rs b/sn_faucet/src/faucet_server.rs index f398e13e53..a337cec1f2 100644 --- a/sn_faucet/src/faucet_server.rs +++ b/sn_faucet/src/faucet_server.rs @@ -132,6 +132,9 @@ async fn respond_to_donate_request( ) -> std::result::Result { let faucet_root = get_faucet_data_dir(); + let permit = semaphore.try_acquire(); + info!("Got donate request with: {transfer_str}"); + let mut wallet = match load_account_wallet_or_create_with_mnemonic(&faucet_root, None) { Ok(wallet) => wallet, Err(_error) => { @@ -142,8 +145,6 @@ async fn respond_to_donate_request( return Ok(response); } }; - let permit = semaphore.try_acquire(); - info!("Got donate request with: {transfer_str}"); // some rate limiting if is_wallet_locked() || permit.is_err() { From 08dc06c2ec8b60d83ab2b34c9b8d36818720029a Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 25 Apr 2024 15:40:24 +0900 Subject: [PATCH 166/205] fix(faucet): cleanup unused vars --- sn_faucet/src/faucet_server.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/sn_faucet/src/faucet_server.rs b/sn_faucet/src/faucet_server.rs index a337cec1f2..ac733148bd 100644 --- a/sn_faucet/src/faucet_server.rs +++ b/sn_faucet/src/faucet_server.rs @@ -261,9 +261,6 @@ async fn startup_server(client: Client) -> Result<()> { // Create a semaphore with a single permit let semaphore = Arc::new(Semaphore::new(1)); - let faucet_root = get_faucet_data_dir(); - let _faucet_wallet = load_account_wallet_or_create_with_mnemonic(&faucet_root, None)?; - #[allow(unused)] let mut balances = HashMap::::new(); #[cfg(feature = "distribution")] From 1cc37d4a8230c696676f4c9e694a194ace9ea9c6 Mon Sep 17 00:00:00 2001 From: qima Date: Mon, 29 Apr 2024 17:28:12 +0800 Subject: [PATCH 167/205] chore: addres review comments --- .../src/bin/subcommands/wallet/hot_wallet.rs | 5 +++- sn_faucet/src/faucet_server.rs | 23 +++++++++---------- sn_transfers/src/wallet/hot_wallet.rs | 1 - 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs b/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs index e04722c238..cd0c8a12e4 100644 --- a/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs +++ b/sn_cli/src/bin/subcommands/wallet/hot_wallet.rs @@ -35,7 +35,10 @@ use std::{path::Path, str::FromStr}; pub enum WalletCmds { /// Print the wallet address. Address { - /// Optional phrase to use for the wallet deriviation from mnemonic entropy. + /// Optional passphrase to protect the mnemonic, + /// it's not the source of the entropy for the mnemonic generation. + /// The mnemonic+passphrase will be the seed. See detail at + /// `` passphrase: Option, }, /// Print the wallet balance. diff --git a/sn_faucet/src/faucet_server.rs b/sn_faucet/src/faucet_server.rs index ac733148bd..01bb805e5a 100644 --- a/sn_faucet/src/faucet_server.rs +++ b/sn_faucet/src/faucet_server.rs @@ -130,11 +130,20 @@ async fn respond_to_donate_request( transfer_str: String, semaphore: Arc, ) -> std::result::Result { - let faucet_root = get_faucet_data_dir(); - let permit = semaphore.try_acquire(); info!("Got donate request with: {transfer_str}"); + // some rate limiting + if is_wallet_locked() || permit.is_err() { + warn!("Rate limited request due"); + let mut response = Response::new("Rate limited".to_string()); + *response.status_mut() = StatusCode::TOO_MANY_REQUESTS; + + // Either opening the file or locking it failed, indicating rate limiting should occur + return Ok(response); + } + + let faucet_root = get_faucet_data_dir(); let mut wallet = match load_account_wallet_or_create_with_mnemonic(&faucet_root, None) { Ok(wallet) => wallet, Err(_error) => { @@ -146,16 +155,6 @@ async fn respond_to_donate_request( } }; - // some rate limiting - if is_wallet_locked() || permit.is_err() { - warn!("Rate limited request due"); - let mut response = Response::new("Rate limited".to_string()); - *response.status_mut() = StatusCode::TOO_MANY_REQUESTS; - - // Either opening the file or locking it failed, indicating rate limiting should occur - return Ok(response); - } - if let Err(err) = fund_faucet_from_genesis_wallet(&client, &mut wallet).await { eprintln!("Failed to load + fund faucet wallet: {err}"); error!("Failed to load + fund faucet wallet: {err}"); diff --git a/sn_transfers/src/wallet/hot_wallet.rs b/sn_transfers/src/wallet/hot_wallet.rs index 2f60d334b6..9e15f59679 100644 --- a/sn_transfers/src/wallet/hot_wallet.rs +++ b/sn_transfers/src/wallet/hot_wallet.rs @@ -62,7 +62,6 @@ impl HotWallet { /// reloads the wallet from disk. fn reload(&mut self) -> Result<()> { - // placeholder random MainSecretKey to take it out let wallet = Self::load_from_path_and_key(self.watchonly_wallet.wallet_dir(), None)?; if *wallet.key.secret_key() != *self.key.secret_key() { From dae7561b37910ff1c328f2b94420e8ed6baccc07 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Mon, 29 Apr 2024 02:57:24 +0530 Subject: [PATCH 168/205] chore: cleanup network events --- sn_networking/src/driver.rs | 4 +-- sn_networking/src/event.rs | 17 +++++------- sn_networking/src/relay_manager.rs | 42 +++++++++++++++++++----------- 3 files changed, 35 insertions(+), 28 deletions(-) diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index dfabc31666..21f5c96923 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -539,7 +539,6 @@ impl NetworkBuilder { connected_peers: 0, bootstrap, relay_manager, - relay_server_reservations: Default::default(), close_group: Default::default(), replication_fetcher, #[cfg(feature = "open-metrics")] @@ -579,6 +578,7 @@ impl NetworkBuilder { pub struct SwarmDriver { pub(crate) swarm: Swarm, pub(crate) self_peer_id: PeerId, + /// When true, we don't filter our local addresses pub(crate) local: bool, pub(crate) is_client: bool, pub(crate) is_behind_home_network: bool, @@ -587,8 +587,6 @@ pub struct SwarmDriver { pub(crate) connected_peers: usize, pub(crate) bootstrap: ContinuousBootstrap, pub(crate) relay_manager: RelayManager, - /// The reservations given out by our relay server - pub(crate) relay_server_reservations: HashSet, /// The peers that are closer to our PeerId. Includes self. pub(crate) close_group: Vec, pub(crate) replication_fetcher: ReplicationFetcher, diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs index 31e04f19c9..76f462051a 100644 --- a/sn_networking/src/event.rs +++ b/sn_networking/src/event.rs @@ -273,7 +273,7 @@ impl SwarmDriver { } = *event { self.relay_manager - .update_on_successful_reservation(&relay_peer_id, &mut self.swarm); + .on_successful_reservation_by_client(&relay_peer_id, &mut self.swarm); } } @@ -287,10 +287,11 @@ impl SwarmDriver { src_peer_id, renewed: _, } => { - self.relay_server_reservations.insert(src_peer_id); + self.relay_manager + .on_successful_reservation_by_server(src_peer_id); } libp2p::relay::Event::ReservationTimedOut { src_peer_id } => { - self.relay_server_reservations.remove(&src_peer_id); + self.relay_manager.on_reservation_timeout(src_peer_id); } _ => {} } @@ -418,12 +419,12 @@ impl SwarmDriver { trace!(%peer_id, ?addrs, "identify: attempting to add addresses to routing table"); // Attempt to add the addresses to the routing table. - for multiaddr in &addrs { + for multiaddr in addrs { let _routing_update = self .swarm .behaviour_mut() .kademlia - .add_address(&peer_id, multiaddr.clone()); + .add_address(&peer_id, multiaddr); } } } @@ -509,7 +510,7 @@ impl SwarmDriver { event_string = "listener closed"; info!("Listener {listener_id:?} with add {addresses:?} has been closed for {reason:?}"); self.relay_manager - .update_on_listener_closed(&listener_id, &mut self.swarm); + .on_listener_closed(&listener_id, &mut self.swarm); } SwarmEvent::IncomingConnection { connection_id, @@ -1312,10 +1313,6 @@ impl SwarmDriver { continue; } - if self.relay_server_reservations.contains(peer_id) { - continue; - } - shall_removed.push((*connection_id, *peer_id)); } diff --git a/sn_networking/src/relay_manager.rs b/sn_networking/src/relay_manager.rs index 720a1fc1cf..6f50d1f0ce 100644 --- a/sn_networking/src/relay_manager.rs +++ b/sn_networking/src/relay_manager.rs @@ -19,8 +19,10 @@ const MAX_POTENTIAL_CANDIDATES: usize = 15; #[derive(Debug)] pub(crate) struct RelayManager { self_peer_id: PeerId, - // states - enabled: bool, + // server states + reserved_by: HashSet, + // client states + enable_client: bool, candidates: VecDeque<(PeerId, Multiaddr)>, waiting_for_reservation: BTreeMap, connected_relays: BTreeMap, @@ -46,7 +48,8 @@ impl RelayManager { .collect(); Self { self_peer_id, - enabled: false, + reserved_by: Default::default(), + enable_client: false, connected_relays: Default::default(), waiting_for_reservation: Default::default(), candidates, @@ -55,13 +58,15 @@ impl RelayManager { } pub(crate) fn enable_hole_punching(&mut self, enable: bool) { - info!("Setting enable hole punching to {enable:?}"); - self.enabled = enable; + info!("Setting relay client mode to {enable:?}"); + self.enable_client = enable; } + /// Should we keep this peer alive? pub(crate) fn keep_alive_peer(&self, peer_id: &PeerId) -> bool { self.connected_relays.contains_key(peer_id) || self.waiting_for_reservation.contains_key(peer_id) + || self.reserved_by.contains(peer_id) } /// Add a potential candidate to the list if it satisfies all the identify checks and also supports the relay server @@ -72,7 +77,6 @@ impl RelayManager { addrs: &HashSet, stream_protocols: &Vec, ) { - trace!("Trying to add potential relay candidates for {peer_id:?} with addrs: {addrs:?}"); if self.candidates.len() >= MAX_POTENTIAL_CANDIDATES { trace!("Got max relay candidates"); return; @@ -88,15 +92,13 @@ impl RelayManager { "Adding {peer_id:?} with {relay_addr:?} as a potential relay candidate" ); self.candidates.push_back((*peer_id, relay_addr)); - } else { - trace!("Was not able to craft relay address"); } } else { - trace!("Addr contains P2pCircuit protocol. Not adding as candidate."); + trace!("Addr for peer {peer_id:?} contains P2pCircuit protocol. Not adding as candidate."); } } } else { - trace!("Peer does not support relay server protocol"); + trace!("Peer {peer_id:?} does not support relay server protocol"); } } @@ -104,7 +106,7 @@ impl RelayManager { /// Try connecting to candidate relays if we are below the threshold connections. /// This is run periodically on a loop. pub(crate) fn try_connecting_to_relay(&mut self, swarm: &mut Swarm) { - if !self.enabled { + if !self.enable_client { return; } @@ -146,8 +148,18 @@ impl RelayManager { } } - /// Update our state after we've successfully made reservation with a relay. - pub(crate) fn update_on_successful_reservation( + /// Update relay server state on incoming reservation from a client + pub(crate) fn on_successful_reservation_by_server(&mut self, peer_id: PeerId) { + self.reserved_by.insert(peer_id); + } + + /// Update relay server state on reservation timeout + pub(crate) fn on_reservation_timeout(&mut self, peer_id: PeerId) { + self.reserved_by.remove(&peer_id); + } + + /// Update client state after we've successfully made reservation with a relay. + pub(crate) fn on_successful_reservation_by_client( &mut self, peer_id: &PeerId, swarm: &mut Swarm, @@ -164,8 +176,8 @@ impl RelayManager { } } - /// Update our state if the reservation has been cancelled or if the relay has closed. - pub(crate) fn update_on_listener_closed( + /// Update client state if the reservation has been cancelled or if the relay has closed. + pub(crate) fn on_listener_closed( &mut self, listener_id: &ListenerId, swarm: &mut Swarm, From b1885903e49d8069cd2ea52ab9ba61be80a901f9 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Mon, 29 Apr 2024 04:26:51 +0530 Subject: [PATCH 169/205] refactor(network): move event handling to its own module --- sn_networking/src/driver.rs | 15 +- sn_networking/src/event.rs | 1564 ------------------- sn_networking/src/event/kad.rs | 585 +++++++ sn_networking/src/event/mod.rs | 273 ++++ sn_networking/src/event/request_response.rs | 394 +++++ sn_networking/src/event/swarm.rs | 644 ++++++++ sn_networking/src/get_record_handler.rs | 300 ---- sn_networking/src/lib.rs | 1 - 8 files changed, 1909 insertions(+), 1867 deletions(-) delete mode 100644 sn_networking/src/event.rs create mode 100644 sn_networking/src/event/kad.rs create mode 100644 sn_networking/src/event/mod.rs create mode 100644 sn_networking/src/event/request_response.rs create mode 100644 sn_networking/src/event/swarm.rs delete mode 100644 sn_networking/src/get_record_handler.rs diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index 21f5c96923..c104ec14cb 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -16,7 +16,6 @@ use crate::{ cmd::SwarmCmd, error::{NetworkError, Result}, event::{NetworkEvent, NodeEvent}, - get_record_handler::PendingGetRecord, multiaddr_pop_p2p, network_discovery::NetworkDiscovery, record_store::{ClientRecordStore, NodeRecordStore, NodeRecordStoreConfig}, @@ -24,7 +23,7 @@ use crate::{ relay_manager::RelayManager, replication_fetcher::ReplicationFetcher, target_arch::{interval, spawn, Instant}, - Network, CLOSE_GROUP_SIZE, + GetRecordError, Network, CLOSE_GROUP_SIZE, }; use crate::{transport, NodeIssue}; use futures::future::Either; @@ -67,6 +66,7 @@ use std::{ use tokio::sync::{mpsc, oneshot}; use tokio::time::Duration; use tracing::warn; +use xor_name::XorName; /// Interval over which we check for the farthest record we _should_ be holding /// based upon our knowledge of the CLOSE_GROUP @@ -85,6 +85,17 @@ pub(crate) enum PendingGetClosestType { } type PendingGetClosest = HashMap)>; +/// Using XorName to differentiate different record content under the same key. +type GetRecordResultMap = HashMap)>; +pub(crate) type PendingGetRecord = HashMap< + QueryId, + ( + oneshot::Sender>, + GetRecordResultMap, + GetRecordCfg, + ), +>; + /// What is the largest packet to send over the network. /// Records larger than this will be rejected. // TODO: revisit once cashnote_redemption is in diff --git a/sn_networking/src/event.rs b/sn_networking/src/event.rs deleted file mode 100644 index 76f462051a..0000000000 --- a/sn_networking/src/event.rs +++ /dev/null @@ -1,1564 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use crate::{ - cmd::SwarmCmd, - driver::{PendingGetClosestType, SwarmDriver}, - error::{NetworkError, Result}, - multiaddr_is_global, multiaddr_strip_p2p, sort_peers_by_address, - target_arch::Instant, - CLOSE_GROUP_SIZE, REPLICATION_PEERS_COUNT, -}; -use core::fmt; -use custom_debug::Debug as CustomDebug; -use itertools::Itertools; -#[cfg(feature = "local-discovery")] -use libp2p::mdns; -use libp2p::{ - kad::{self, GetClosestPeersError, InboundRequest, QueryResult, Record, RecordKey, K_VALUE}, - multiaddr::Protocol, - request_response::{self, Message, ResponseChannel as PeerResponseChannel}, - swarm::{ - dial_opts::{DialOpts, PeerCondition}, - DialError, SwarmEvent, - }, - Multiaddr, PeerId, TransportError, -}; -use rand::{rngs::OsRng, Rng}; -use sn_protocol::version::{IDENTIFY_NODE_VERSION_STR, IDENTIFY_PROTOCOL_STR}; -use sn_protocol::{ - get_port_from_multiaddr, - messages::{CmdResponse, Query, Request, Response}, - storage::RecordType, - NetworkAddress, PrettyPrintRecordKey, -}; -use sn_transfers::PaymentQuote; -use std::{ - collections::{hash_map::Entry, BTreeSet, HashSet}, - fmt::{Debug, Formatter}, -}; -use tokio::sync::oneshot; -use tokio::time::Duration; -use tracing::{info, warn}; - -/// NodeEvent enum -#[derive(CustomDebug)] -pub(super) enum NodeEvent { - MsgReceived(request_response::Event), - Kademlia(kad::Event), - #[cfg(feature = "local-discovery")] - Mdns(Box), - Identify(Box), - Dcutr(Box), - RelayClient(Box), - RelayServer(Box), -} - -impl From> for NodeEvent { - fn from(event: request_response::Event) -> Self { - NodeEvent::MsgReceived(event) - } -} - -impl From for NodeEvent { - fn from(event: kad::Event) -> Self { - NodeEvent::Kademlia(event) - } -} - -#[cfg(feature = "local-discovery")] -impl From for NodeEvent { - fn from(event: mdns::Event) -> Self { - NodeEvent::Mdns(Box::new(event)) - } -} - -impl From for NodeEvent { - fn from(event: libp2p::identify::Event) -> Self { - NodeEvent::Identify(Box::new(event)) - } -} -impl From for NodeEvent { - fn from(event: libp2p::dcutr::Event) -> Self { - NodeEvent::Dcutr(Box::new(event)) - } -} -impl From for NodeEvent { - fn from(event: libp2p::relay::client::Event) -> Self { - NodeEvent::RelayClient(Box::new(event)) - } -} -impl From for NodeEvent { - fn from(event: libp2p::relay::Event) -> Self { - NodeEvent::RelayServer(Box::new(event)) - } -} - -#[derive(CustomDebug)] -/// Channel to send the `Response` through. -pub enum MsgResponder { - /// Respond to a request from `self` through a simple one-shot channel. - FromSelf(Option>>), - /// Respond to a request from a peer in the network. - FromPeer(PeerResponseChannel), -} - -#[allow(clippy::large_enum_variant)] -/// Events forwarded by the underlying Network; to be used by the upper layers -pub enum NetworkEvent { - /// Incoming `Query` from a peer - QueryRequestReceived { - /// Query - query: Query, - /// The channel to send the `Response` through - channel: MsgResponder, - }, - /// Handles the responses that are not awaited at the call site - ResponseReceived { - /// Response - res: Response, - }, - /// Peer has been added to the Routing Table. And the number of connected peers. - PeerAdded(PeerId, usize), - /// Peer has been removed from the Routing Table. And the number of connected peers. - PeerRemoved(PeerId, usize), - /// The peer does not support our protocol - PeerWithUnsupportedProtocol { - our_protocol: String, - their_protocol: String, - }, - /// The peer is now considered as a bad node, due to the detected bad behaviour - PeerConsideredAsBad { - detected_by: PeerId, - bad_peer: PeerId, - bad_behaviour: String, - }, - /// The records bearing these keys are to be fetched from the holder or the network - KeysToFetchForReplication(Vec<(PeerId, RecordKey)>), - /// Started listening on a new address - NewListenAddr(Multiaddr), - /// Report unverified record - UnverifiedRecord(Record), - /// Terminate Node on unrecoverable errors - TerminateNode { reason: TerminateNodeReason }, - /// List of peer nodes that failed to fetch replication copy from. - FailedToFetchHolders(BTreeSet), - /// A peer in RT that supposed to be verified. - BadNodeVerification { peer_id: PeerId }, - /// Quotes to be verified - QuoteVerification { quotes: Vec<(PeerId, PaymentQuote)> }, - /// Carry out chunk proof check against the specified record and peer - ChunkProofVerification { - peer_id: PeerId, - keys_to_verify: Vec, - }, -} - -/// Terminate node for the following reason -#[derive(Debug, Clone)] -pub enum TerminateNodeReason { - HardDiskWriteError, -} - -// Manually implement Debug as `#[debug(with = "unverified_record_fmt")]` not working as expected. -impl Debug for NetworkEvent { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - match self { - NetworkEvent::QueryRequestReceived { query, .. } => { - write!(f, "NetworkEvent::QueryRequestReceived({query:?})") - } - NetworkEvent::ResponseReceived { res, .. } => { - write!(f, "NetworkEvent::ResponseReceived({res:?})") - } - NetworkEvent::PeerAdded(peer_id, connected_peers) => { - write!(f, "NetworkEvent::PeerAdded({peer_id:?}, {connected_peers})") - } - NetworkEvent::PeerRemoved(peer_id, connected_peers) => { - write!( - f, - "NetworkEvent::PeerRemoved({peer_id:?}, {connected_peers})" - ) - } - NetworkEvent::PeerWithUnsupportedProtocol { - our_protocol, - their_protocol, - } => { - write!(f, "NetworkEvent::PeerWithUnsupportedProtocol({our_protocol:?}, {their_protocol:?})") - } - NetworkEvent::PeerConsideredAsBad { - bad_peer, - bad_behaviour, - .. - } => { - write!( - f, - "NetworkEvent::PeerConsideredAsBad({bad_peer:?}, {bad_behaviour:?})" - ) - } - NetworkEvent::KeysToFetchForReplication(list) => { - let keys_len = list.len(); - write!(f, "NetworkEvent::KeysForReplication({keys_len:?})") - } - NetworkEvent::NewListenAddr(addr) => { - write!(f, "NetworkEvent::NewListenAddr({addr:?})") - } - NetworkEvent::UnverifiedRecord(record) => { - let pretty_key = PrettyPrintRecordKey::from(&record.key); - write!(f, "NetworkEvent::UnverifiedRecord({pretty_key:?})") - } - NetworkEvent::TerminateNode { reason } => { - write!(f, "NetworkEvent::TerminateNode({reason:?})") - } - NetworkEvent::FailedToFetchHolders(bad_nodes) => { - write!(f, "NetworkEvent::FailedToFetchHolders({bad_nodes:?})") - } - NetworkEvent::BadNodeVerification { peer_id } => { - write!(f, "NetworkEvent::BadNodeVerification({peer_id:?})") - } - NetworkEvent::QuoteVerification { quotes } => { - write!( - f, - "NetworkEvent::QuoteVerification({} quotes)", - quotes.len() - ) - } - NetworkEvent::ChunkProofVerification { - peer_id, - keys_to_verify, - } => { - write!( - f, - "NetworkEvent::ChunkProofVerification({peer_id:?} {keys_to_verify:?})" - ) - } - } - } -} - -impl SwarmDriver { - /// Handle `SwarmEvents` - pub(super) fn handle_swarm_events(&mut self, event: SwarmEvent) -> Result<()> { - let start = Instant::now(); - let event_string; - match event { - SwarmEvent::Behaviour(NodeEvent::MsgReceived(event)) => { - event_string = "msg_received"; - if let Err(e) = self.handle_msg(event) { - warn!("MsgReceivedError: {e:?}"); - } - } - SwarmEvent::Behaviour(NodeEvent::Kademlia(kad_event)) => { - event_string = "kad_event"; - self.handle_kad_event(kad_event)?; - } - SwarmEvent::Behaviour(NodeEvent::Dcutr(event)) => { - event_string = "dcutr_event"; - info!( - "Dcutr with remote peer: {:?} is: {:?}", - event.remote_peer_id, event.result - ); - } - SwarmEvent::Behaviour(NodeEvent::RelayClient(event)) => { - event_string = "relay_client_event"; - - info!(?event, "relay client event"); - - if let libp2p::relay::client::Event::ReservationReqAccepted { - relay_peer_id, .. - } = *event - { - self.relay_manager - .on_successful_reservation_by_client(&relay_peer_id, &mut self.swarm); - } - } - - SwarmEvent::Behaviour(NodeEvent::RelayServer(event)) => { - event_string = "relay_server_event"; - - info!(?event, "relay server event"); - - match *event { - libp2p::relay::Event::ReservationReqAccepted { - src_peer_id, - renewed: _, - } => { - self.relay_manager - .on_successful_reservation_by_server(src_peer_id); - } - libp2p::relay::Event::ReservationTimedOut { src_peer_id } => { - self.relay_manager.on_reservation_timeout(src_peer_id); - } - _ => {} - } - } - SwarmEvent::Behaviour(NodeEvent::Identify(iden)) => { - event_string = "identify"; - - match *iden { - libp2p::identify::Event::Received { peer_id, info } => { - trace!(%peer_id, ?info, "identify: received info"); - - if info.protocol_version != IDENTIFY_PROTOCOL_STR.to_string() { - warn!(?info.protocol_version, "identify: {peer_id:?} does not have the same protocol. Our IDENTIFY_PROTOCOL_STR: {:?}", IDENTIFY_PROTOCOL_STR.as_str()); - - self.send_event(NetworkEvent::PeerWithUnsupportedProtocol { - our_protocol: IDENTIFY_PROTOCOL_STR.to_string(), - their_protocol: info.protocol_version, - }); - - return Ok(()); - } - - // if client, return. - if info.agent_version != IDENTIFY_NODE_VERSION_STR.to_string() { - return Ok(()); - } - - let has_dialed = self.dialed_peers.contains(&peer_id); - - // If we're not in local mode, only add globally reachable addresses. - // Strip the `/p2p/...` part of the multiaddresses. - // Collect into a HashSet directly to avoid multiple allocations and handle deduplication. - let addrs: HashSet = match self.local { - true => info - .listen_addrs - .into_iter() - .map(|addr| multiaddr_strip_p2p(&addr)) - .collect(), - false => info - .listen_addrs - .into_iter() - .filter(multiaddr_is_global) - .map(|addr| multiaddr_strip_p2p(&addr)) - .collect(), - }; - - self.relay_manager.add_potential_candidates( - &peer_id, - &addrs, - &info.protocols, - ); - - // When received an identify from un-dialed peer, try to dial it - // The dial shall trigger the same identify to be sent again and confirm - // peer is external accessible, hence safe to be added into RT. - if !self.local && !has_dialed { - // Only need to dial back for not fulfilled kbucket - let (kbucket_full, already_present_in_rt, ilog2) = - if let Some(kbucket) = - self.swarm.behaviour_mut().kademlia.kbucket(peer_id) - { - let ilog2 = kbucket.range().0.ilog2(); - let num_peers = kbucket.num_entries(); - let mut is_bucket_full = num_peers >= K_VALUE.into(); - - // check if peer_id is already a part of RT - let already_present_in_rt = kbucket - .iter() - .any(|entry| entry.node.key.preimage() == &peer_id); - - // If the bucket contains any of a bootstrap node, - // consider the bucket is not full and dial back - // so that the bootstrap nodes can be replaced. - if is_bucket_full { - if let Some(peers) = self.bootstrap_peers.get(&ilog2) { - if kbucket.iter().any(|entry| { - peers.contains(entry.node.key.preimage()) - }) { - is_bucket_full = false; - } - } - } - - (is_bucket_full, already_present_in_rt, ilog2) - } else { - return Ok(()); - }; - - if kbucket_full { - trace!("received identify for a full bucket {ilog2:?}, not dialing {peer_id:?} on {addrs:?}"); - return Ok(()); - } else if already_present_in_rt { - trace!("received identify for {peer_id:?} that is already part of the RT. Not dialing {peer_id:?} on {addrs:?}"); - return Ok(()); - } - - info!(%peer_id, ?addrs, "received identify info from undialed peer for not full kbucket {ilog2:?}, dial back to confirm external accessible"); - if let Err(err) = self.swarm.dial( - DialOpts::peer_id(peer_id) - .condition(PeerCondition::NotDialing) - .addresses(addrs.iter().cloned().collect()) - .build(), - ) { - warn!(%peer_id, ?addrs, "dialing error: {err:?}"); - } - - trace!( - "SwarmEvent handled in {:?}: {event_string:?}", - start.elapsed() - ); - return Ok(()); - } - - // If we are not local, we care only for peers that we dialed and thus are reachable. - if self.local || has_dialed { - // To reduce the bad_node check resource usage, - // during the connection establish process, only check cached black_list - // The periodical check, which involves network queries shall filter - // out bad_nodes eventually. - if let Some((_issues, true)) = self.bad_nodes.get(&peer_id) { - info!("Peer {peer_id:?} is considered as bad, blocking it."); - } else { - self.remove_bootstrap_from_full(peer_id); - - trace!(%peer_id, ?addrs, "identify: attempting to add addresses to routing table"); - - // Attempt to add the addresses to the routing table. - for multiaddr in addrs { - let _routing_update = self - .swarm - .behaviour_mut() - .kademlia - .add_address(&peer_id, multiaddr); - } - } - } - trace!( - "SwarmEvent handled in {:?}: {event_string:?}", - start.elapsed() - ); - } - // Log the other Identify events. - libp2p::identify::Event::Sent { .. } => trace!("identify: {iden:?}"), - libp2p::identify::Event::Pushed { .. } => trace!("identify: {iden:?}"), - libp2p::identify::Event::Error { .. } => trace!("identify: {iden:?}"), - } - } - #[cfg(feature = "local-discovery")] - SwarmEvent::Behaviour(NodeEvent::Mdns(mdns_event)) => { - event_string = "mdns"; - match *mdns_event { - mdns::Event::Discovered(list) => { - if self.local { - for (peer_id, addr) in list { - // The multiaddr does not contain the peer ID, so add it. - let addr = addr.with(Protocol::P2p(peer_id)); - - info!(%addr, "mDNS node discovered and dialing"); - - if let Err(err) = self.dial(addr.clone()) { - warn!(%addr, "mDNS node dial error: {err:?}"); - } - } - } - } - mdns::Event::Expired(peer) => { - trace!("mdns peer {peer:?} expired"); - } - } - } - - SwarmEvent::NewListenAddr { - address, - listener_id, - } => { - event_string = "new listen addr"; - - // update our stored port if it is configured to be 0 or None - match self.listen_port { - Some(0) | None => { - if let Some(actual_port) = get_port_from_multiaddr(&address) { - info!("Our listen port is configured as 0 or is not set. Setting it to our actual port: {actual_port}"); - self.listen_port = Some(actual_port); - } - } - _ => {} - }; - - let local_peer_id = *self.swarm.local_peer_id(); - let address = address.with(Protocol::P2p(local_peer_id)); - - // Trigger server mode if we're not a client and we should not add our own address if we're behind - // home network. - if !self.is_client && !self.is_behind_home_network { - if self.local { - // all addresses are effectively external here... - // this is needed for Kad Mode::Server - self.swarm.add_external_address(address.clone()); - } else { - // only add our global addresses - if multiaddr_is_global(&address) { - self.swarm.add_external_address(address.clone()); - } - } - } - - self.send_event(NetworkEvent::NewListenAddr(address.clone())); - - info!("Local node is listening {listener_id:?} on {address:?}"); - } - SwarmEvent::ListenerClosed { - listener_id, - addresses, - reason, - } => { - event_string = "listener closed"; - info!("Listener {listener_id:?} with add {addresses:?} has been closed for {reason:?}"); - self.relay_manager - .on_listener_closed(&listener_id, &mut self.swarm); - } - SwarmEvent::IncomingConnection { - connection_id, - local_addr, - send_back_addr, - } => { - event_string = "incoming"; - trace!("IncomingConnection ({connection_id:?}) with local_addr: {local_addr:?} send_back_addr: {send_back_addr:?}"); - } - SwarmEvent::ConnectionEstablished { - peer_id, - endpoint, - num_established, - connection_id, - concurrent_dial_errors, - established_in, - } => { - event_string = "ConnectionEstablished"; - trace!(%peer_id, num_established, ?concurrent_dial_errors, "ConnectionEstablished ({connection_id:?}) in {established_in:?}: {}", endpoint_str(&endpoint)); - - let _ = self.live_connected_peers.insert( - connection_id, - (peer_id, Instant::now() + Duration::from_secs(60)), - ); - - if endpoint.is_dialer() { - self.dialed_peers.push(peer_id); - } - } - SwarmEvent::ConnectionClosed { - peer_id, - endpoint, - cause, - num_established, - connection_id, - } => { - event_string = "ConnectionClosed"; - trace!(%peer_id, ?connection_id, ?cause, num_established, "ConnectionClosed: {}", endpoint_str(&endpoint)); - let _ = self.live_connected_peers.remove(&connection_id); - } - SwarmEvent::OutgoingConnectionError { - connection_id, - peer_id: None, - error, - } => { - event_string = "OutgoingConnErr"; - warn!("OutgoingConnectionError to on {connection_id:?} - {error:?}"); - } - SwarmEvent::OutgoingConnectionError { - peer_id: Some(failed_peer_id), - error, - connection_id, - } => { - event_string = "OutgoingConnErr"; - warn!("OutgoingConnectionError to {failed_peer_id:?} on {connection_id:?} - {error:?}"); - - // we need to decide if this was a critical error and the peer should be removed from the routing table - let should_clean_peer = match error { - DialError::Transport(errors) => { - // as it's an outgoing error, if it's transport based we can assume it is _our_ fault - // - // (eg, could not get a port for a tcp connection) - // so we default to it not being a real issue - // unless there are _specific_ errors (connection refused eg) - error!("Dial errors len : {:?}", errors.len()); - let mut there_is_a_serious_issue = false; - for (_addr, err) in errors { - error!("OutgoingTransport error : {err:?}"); - - match err { - TransportError::MultiaddrNotSupported(addr) => { - warn!("Multiaddr not supported : {addr:?}"); - // if we can't dial a peer on a given address, we should remove it from the routing table - there_is_a_serious_issue = true - } - TransportError::Other(err) => { - let problematic_errors = [ - "ConnectionRefused", - "HostUnreachable", - "HandshakeTimedOut", - ]; - - let is_bootstrap_peer = self - .bootstrap_peers - .iter() - .any(|(_ilog2, peers)| peers.contains(&failed_peer_id)); - - if is_bootstrap_peer - && self.connected_peers < self.bootstrap_peers.len() - { - warn!("OutgoingConnectionError: On bootstrap peer {failed_peer_id:?}, while still in bootstrap mode, ignoring"); - there_is_a_serious_issue = false; - } else { - // It is really difficult to match this error, due to being eg: - // Custom { kind: Other, error: Left(Left(Os { code: 61, kind: ConnectionRefused, message: "Connection refused" })) } - // if we can match that, let's. But meanwhile we'll check the message - let error_msg = format!("{err:?}"); - if problematic_errors - .iter() - .any(|err| error_msg.contains(err)) - { - warn!("Problematic error encountered: {error_msg}"); - there_is_a_serious_issue = true; - } - } - } - } - } - there_is_a_serious_issue - } - DialError::NoAddresses => { - // We provided no address, and while we can't really blame the peer - // we also can't connect, so we opt to cleanup... - warn!("OutgoingConnectionError: No address provided"); - true - } - DialError::Aborted => { - // not their fault - warn!("OutgoingConnectionError: Aborted"); - false - } - DialError::DialPeerConditionFalse(_) => { - // we could not dial due to an internal condition, so not their issue - warn!("OutgoingConnectionError: DialPeerConditionFalse"); - false - } - DialError::LocalPeerId { endpoint, .. } => { - // This is actually _us_ So we should remove this from the RT - error!( - "OutgoingConnectionError: LocalPeerId: {}", - endpoint_str(&endpoint) - ); - true - } - DialError::WrongPeerId { obtained, endpoint } => { - // The peer id we attempted to dial was not the one we expected - // cleanup - error!("OutgoingConnectionError: WrongPeerId: obtained: {obtained:?}, endpoint: {endpoint:?}"); - true - } - DialError::Denied { cause } => { - // The peer denied our connection - // cleanup - error!("OutgoingConnectionError: Denied: {cause:?}"); - true - } - }; - - if should_clean_peer { - warn!("Tracking issue of {failed_peer_id:?}. Clearing it out for now"); - - if let Some(dead_peer) = self - .swarm - .behaviour_mut() - .kademlia - .remove_peer(&failed_peer_id) - { - self.connected_peers = self.connected_peers.saturating_sub(1); - - self.handle_cmd(SwarmCmd::RecordNodeIssue { - peer_id: failed_peer_id, - issue: crate::NodeIssue::ConnectionIssue, - })?; - - self.send_event(NetworkEvent::PeerRemoved( - *dead_peer.node.key.preimage(), - self.connected_peers, - )); - - self.log_kbuckets(&failed_peer_id); - let _ = self.check_for_change_in_our_close_group(); - } - } - } - SwarmEvent::IncomingConnectionError { - connection_id, - local_addr, - send_back_addr, - error, - } => { - event_string = "Incoming ConnErr"; - error!("IncomingConnectionError from local_addr:?{local_addr:?}, send_back_addr {send_back_addr:?} on {connection_id:?} with error {error:?}"); - } - SwarmEvent::Dialing { - peer_id, - connection_id, - } => { - event_string = "Dialing"; - trace!("Dialing {peer_id:?} on {connection_id:?}"); - } - SwarmEvent::NewExternalAddrCandidate { address } => { - event_string = "NewExternalAddrCandidate"; - - if !self.swarm.external_addresses().any(|addr| addr == &address) - && !self.is_client - // If we are behind a home network, then our IP is returned here. We should be only having - // relay server as our external address - // todo: can our relay address be reported here? If so, maybe we should add them. - && !self.is_behind_home_network - { - debug!(%address, "external address: new candidate"); - - // Identify will let us know when we have a candidate. (Peers will tell us what address they see us as.) - // We manually confirm this to be our externally reachable address, though in theory it's possible we - // are not actually reachable. This event returns addresses with ports that were not set by the user, - // so we must not add those ports as they will not be forwarded. - // Setting this will also switch kad to server mode if it's not already in it. - if let Some(our_port) = self.listen_port { - if let Some(port) = get_port_from_multiaddr(&address) { - if port == our_port { - info!(%address, "external address: new candidate has the same configured port, adding it."); - self.swarm.add_external_address(address); - } else { - info!(%address, %our_port, "external address: new candidate has a different port, not adding it."); - } - } - } else { - trace!("external address: listen port not set. This has to be set if you're running a node"); - } - } - let all_external_addresses = self.swarm.external_addresses().collect_vec(); - let all_listeners = self.swarm.listeners().collect_vec(); - debug!("All our listeners: {all_listeners:?}"); - debug!("All our external addresses: {all_external_addresses:?}"); - } - SwarmEvent::ExternalAddrConfirmed { address } => { - event_string = "ExternalAddrConfirmed"; - info!(%address, "external address: confirmed"); - } - SwarmEvent::ExternalAddrExpired { address } => { - event_string = "ExternalAddrExpired"; - info!(%address, "external address: expired"); - } - other => { - event_string = "Other"; - - trace!("SwarmEvent has been ignored: {other:?}") - } - } - self.remove_outdated_connections(); - - self.log_handling(event_string.to_string(), start.elapsed()); - - trace!( - "SwarmEvent handled in {:?}: {event_string:?}", - start.elapsed() - ); - Ok(()) - } - - /// Forwards `Request` to the upper layers using `Sender`. Sends `Response` to the peers - pub fn handle_msg( - &mut self, - event: request_response::Event, - ) -> Result<(), NetworkError> { - match event { - request_response::Event::Message { message, peer } => match message { - Message::Request { - request, - channel, - request_id, - .. - } => { - trace!("Received request {request_id:?} from peer {peer:?}, req: {request:?}"); - // If the request is replication or quote verification, - // we can handle it and send the OK response here. - // As the handle result is unimportant to the sender. - match request { - Request::Cmd(sn_protocol::messages::Cmd::Replicate { holder, keys }) => { - let response = Response::Cmd( - sn_protocol::messages::CmdResponse::Replicate(Ok(())), - ); - self.swarm - .behaviour_mut() - .request_response - .send_response(channel, response) - .map_err(|_| NetworkError::InternalMsgChannelDropped)?; - - self.add_keys_to_replication_fetcher(holder, keys); - } - Request::Cmd(sn_protocol::messages::Cmd::QuoteVerification { - quotes, - .. - }) => { - let response = Response::Cmd( - sn_protocol::messages::CmdResponse::QuoteVerification(Ok(())), - ); - self.swarm - .behaviour_mut() - .request_response - .send_response(channel, response) - .map_err(|_| NetworkError::InternalMsgChannelDropped)?; - - // The keypair is required to verify the quotes, - // hence throw it up to Network layer for further actions. - let quotes = quotes - .iter() - .filter_map(|(peer_address, quote)| { - peer_address - .as_peer_id() - .map(|peer_id| (peer_id, quote.clone())) - }) - .collect(); - self.send_event(NetworkEvent::QuoteVerification { quotes }) - } - Request::Cmd(sn_protocol::messages::Cmd::PeerConsideredAsBad { - detected_by, - bad_peer, - bad_behaviour, - }) => { - let response = Response::Cmd( - sn_protocol::messages::CmdResponse::PeerConsideredAsBad(Ok(())), - ); - self.swarm - .behaviour_mut() - .request_response - .send_response(channel, response) - .map_err(|_| NetworkError::InternalMsgChannelDropped)?; - - if bad_peer == NetworkAddress::from_peer(self.self_peer_id) { - warn!("Peer {detected_by:?} consider us as BAD, due to {bad_behaviour:?}."); - // TODO: shall we terminate self after received such notifications - // from the majority close_group nodes around us? - } else { - error!("Received a bad_peer notification from {detected_by:?}, targeting {bad_peer:?}, which is not us."); - } - } - Request::Query(query) => { - self.send_event(NetworkEvent::QueryRequestReceived { - query, - channel: MsgResponder::FromPeer(channel), - }) - } - } - } - Message::Response { - request_id, - response, - } => { - trace!("Got response {request_id:?} from peer {peer:?}, res: {response}."); - if let Some(sender) = self.pending_requests.remove(&request_id) { - // The sender will be provided if the caller (Requester) is awaiting for a response - // at the call site. - // Else the Request was just sent to the peer and the Response was - // meant to be handled in another way and is not awaited. - match sender { - Some(sender) => sender - .send(Ok(response)) - .map_err(|_| NetworkError::InternalMsgChannelDropped)?, - None => { - if let Response::Cmd(CmdResponse::Replicate(Ok(()))) = response { - // Nothing to do, response was fine - // This only exists to ensure we dont drop the handle and - // exit early, potentially logging false connection woes - } else { - // responses that are not awaited at the call site must be handled - // separately - self.send_event(NetworkEvent::ResponseReceived { - res: response, - }); - } - } - } - } else { - warn!("Tried to remove a RequestId from pending_requests which was not inserted in the first place. - Use Cmd::SendRequest with sender:None if you want the Response to be fed into the common handle_response function"); - } - } - }, - request_response::Event::OutboundFailure { - request_id, - error, - peer, - } => { - if let Some(sender) = self.pending_requests.remove(&request_id) { - match sender { - Some(sender) => { - sender - .send(Err(error.into())) - .map_err(|_| NetworkError::InternalMsgChannelDropped)?; - } - None => { - warn!("RequestResponse: OutboundFailure for request_id: {request_id:?} and peer: {peer:?}, with error: {error:?}"); - return Err(NetworkError::ReceivedResponseDropped(request_id)); - } - } - } else { - warn!("RequestResponse: OutboundFailure for request_id: {request_id:?} and peer: {peer:?}, with error: {error:?}"); - return Err(NetworkError::ReceivedResponseDropped(request_id)); - } - } - request_response::Event::InboundFailure { - peer, - request_id, - error, - } => { - warn!("RequestResponse: InboundFailure for request_id: {request_id:?} and peer: {peer:?}, with error: {error:?}"); - } - request_response::Event::ResponseSent { peer, request_id } => { - trace!("ResponseSent for request_id: {request_id:?} and peer: {peer:?}"); - } - } - Ok(()) - } - - fn handle_kad_event(&mut self, kad_event: kad::Event) -> Result<()> { - let start = Instant::now(); - let event_string; - - match kad_event { - kad::Event::OutboundQueryProgressed { - id, - result: QueryResult::GetClosestPeers(Ok(ref closest_peers)), - ref stats, - ref step, - } => { - event_string = "kad_event::get_closest_peers"; - trace!( - "Query task {id:?} of key {:?} returned with peers {:?}, {stats:?} - {step:?}", - hex::encode(closest_peers.key.clone()), - closest_peers.peers, - ); - - if let Entry::Occupied(mut entry) = self.pending_get_closest_peers.entry(id) { - let (_, current_closest) = entry.get_mut(); - - // TODO: consider order the result and terminate when reach any of the - // following criteria: - // 1, `stats.num_pending()` is 0 - // 2, `stats.duration()` is longer than a defined period - current_closest.extend(closest_peers.peers.clone()); - if current_closest.len() >= usize::from(K_VALUE) || step.last { - let (get_closest_type, current_closest) = entry.remove(); - match get_closest_type { - PendingGetClosestType::NetworkDiscovery => self - .network_discovery - .handle_get_closest_query(current_closest), - PendingGetClosestType::FunctionCall(sender) => { - sender - .send(current_closest) - .map_err(|_| NetworkError::InternalMsgChannelDropped)?; - } - } - } - } else { - trace!("Can't locate query task {id:?}, it has likely been completed already."); - return Err(NetworkError::ReceivedKademliaEventDropped { - query_id: id, - event: "GetClosestPeers Ok".to_string(), - }); - } - } - // Handle GetClosestPeers timeouts - kad::Event::OutboundQueryProgressed { - id, - result: QueryResult::GetClosestPeers(Err(ref err)), - ref stats, - ref step, - } => { - event_string = "kad_event::get_closest_peers_err"; - error!("GetClosest Query task {id:?} errored with {err:?}, {stats:?} - {step:?}"); - - let (get_closest_type, mut current_closest) = - self.pending_get_closest_peers.remove(&id).ok_or_else(|| { - trace!( - "Can't locate query task {id:?}, it has likely been completed already." - ); - NetworkError::ReceivedKademliaEventDropped { - query_id: id, - event: "Get ClosestPeers error".to_string(), - } - })?; - - // We have `current_closest` from previous progress, - // and `peers` from `GetClosestPeersError`. - // Trust them and leave for the caller to check whether they are enough. - match err { - GetClosestPeersError::Timeout { ref peers, .. } => { - current_closest.extend(peers); - } - } - - match get_closest_type { - PendingGetClosestType::NetworkDiscovery => self - .network_discovery - .handle_get_closest_query(current_closest), - PendingGetClosestType::FunctionCall(sender) => { - sender - .send(current_closest) - .map_err(|_| NetworkError::InternalMsgChannelDropped)?; - } - } - } - - kad::Event::OutboundQueryProgressed { - id, - result: QueryResult::GetRecord(Ok(kad::GetRecordOk::FoundRecord(peer_record))), - stats, - step, - } => { - event_string = "kad_event::get_record::found"; - trace!( - "Query task {id:?} returned with record {:?} from peer {:?}, {stats:?} - {step:?}", - PrettyPrintRecordKey::from(&peer_record.record.key), - peer_record.peer - ); - self.accumulate_get_record_found(id, peer_record, stats, step)?; - } - kad::Event::OutboundQueryProgressed { - id, - result: - QueryResult::GetRecord(Ok(kad::GetRecordOk::FinishedWithNoAdditionalRecord { - cache_candidates, - })), - stats, - step, - } => { - event_string = "kad_event::get_record::finished_no_additional"; - trace!("Query task {id:?} of get_record completed with {stats:?} - {step:?} - {cache_candidates:?}"); - self.handle_get_record_finished(id, step)?; - } - kad::Event::OutboundQueryProgressed { - id, - result: QueryResult::GetRecord(Err(get_record_err)), - stats, - step, - } => { - // log the errors - match &get_record_err { - kad::GetRecordError::NotFound { key, closest_peers } => { - event_string = "kad_event::GetRecordError::NotFound"; - info!("Query task {id:?} NotFound record {:?} among peers {closest_peers:?}, {stats:?} - {step:?}", - PrettyPrintRecordKey::from(key)); - } - kad::GetRecordError::QuorumFailed { - key, - records, - quorum, - } => { - event_string = "kad_event::GetRecordError::QuorumFailed"; - let pretty_key = PrettyPrintRecordKey::from(key); - let peers = records - .iter() - .map(|peer_record| peer_record.peer) - .collect_vec(); - info!("Query task {id:?} QuorumFailed record {pretty_key:?} among peers {peers:?} with quorum {quorum:?}, {stats:?} - {step:?}"); - } - kad::GetRecordError::Timeout { key } => { - event_string = "kad_event::GetRecordError::Timeout"; - let pretty_key = PrettyPrintRecordKey::from(key); - - debug!( - "Query task {id:?} timed out when looking for record {pretty_key:?}" - ); - } - } - self.handle_get_record_error(id, get_record_err, stats, step)?; - } - kad::Event::OutboundQueryProgressed { - id, - result: QueryResult::PutRecord(Err(put_record_err)), - stats, - step, - } => { - // Currently, only `client` calls `put_record_to` to upload data. - // The result of such operation is not critical to client in general. - // However, if client keeps receiving error responses, it may indicating: - // 1, Client itself is with slow connection - // OR - // 2, The payee node selected could be in trouble - // - // TODO: Figure out which payee node the error response is related to, - // and may exclude that node from later on payee selection. - let (key, success, quorum) = match &put_record_err { - kad::PutRecordError::QuorumFailed { - key, - success, - quorum, - } => { - event_string = "kad_event::PutRecordError::QuorumFailed"; - (key, success, quorum) - } - kad::PutRecordError::Timeout { - key, - success, - quorum, - } => { - event_string = "kad_event::PutRecordError::Timeout"; - (key, success, quorum) - } - }; - error!("Query task {id:?} failed put record {:?} {:?}, required quorum {quorum}, stored on {success:?}, {stats:?} - {step:?}", - PrettyPrintRecordKey::from(key), event_string); - } - kad::Event::OutboundQueryProgressed { - id, - result: QueryResult::PutRecord(Ok(put_record_ok)), - stats, - step, - } => { - event_string = "kad_event::PutRecordOk"; - trace!( - "Query task {id:?} put record {:?} ok, {stats:?} - {step:?}", - PrettyPrintRecordKey::from(&put_record_ok.key) - ); - } - // Shall no longer receive this event - kad::Event::OutboundQueryProgressed { - id, - result: QueryResult::Bootstrap(bootstrap_result), - step, - .. - } => { - event_string = "kad_event::OutboundQueryProgressed::Bootstrap"; - // here BootstrapOk::num_remaining refers to the remaining random peer IDs to query, one per - // bucket that still needs refreshing. - trace!("Kademlia Bootstrap with {id:?} progressed with {bootstrap_result:?} and step {step:?}"); - } - kad::Event::RoutingUpdated { - peer, - is_new_peer, - old_peer, - .. - } => { - event_string = "kad_event::RoutingUpdated"; - if is_new_peer { - self.connected_peers = self.connected_peers.saturating_add(1); - - info!("New peer added to routing table: {peer:?}, now we have #{} connected peers", self.connected_peers); - self.log_kbuckets(&peer); - - // This should only happen once - if self.bootstrap.notify_new_peer() { - info!("Performing the first bootstrap"); - self.trigger_network_discovery(); - } - self.send_event(NetworkEvent::PeerAdded(peer, self.connected_peers)); - } - - info!("kad_event::RoutingUpdated {:?}: {peer:?}, is_new_peer: {is_new_peer:?} old_peer: {old_peer:?}", self.connected_peers); - if old_peer.is_some() { - self.connected_peers = self.connected_peers.saturating_sub(1); - - info!("Evicted old peer on new peer join: {old_peer:?}"); - self.send_event(NetworkEvent::PeerRemoved(peer, self.connected_peers)); - self.log_kbuckets(&peer); - } - let _ = self.check_for_change_in_our_close_group(); - } - kad::Event::InboundRequest { - request: InboundRequest::PutRecord { .. }, - } => { - event_string = "kad_event::InboundRequest::PutRecord"; - // Ignored to reduce logging. When `Record filtering` is enabled, - // the `record` variable will contain the content for further validation before put. - } - kad::Event::InboundRequest { - request: InboundRequest::FindNode { .. }, - } => { - event_string = "kad_event::InboundRequest::FindNode"; - // Ignored to reduce logging. With continuous bootstrap, this is triggered often. - } - kad::Event::InboundRequest { - request: - InboundRequest::GetRecord { - num_closer_peers, - present_locally, - }, - } => { - event_string = "kad_event::InboundRequest::GetRecord"; - if !present_locally && num_closer_peers < CLOSE_GROUP_SIZE { - trace!("InboundRequest::GetRecord doesn't have local record, with {num_closer_peers:?} closer_peers"); - } - } - kad::Event::UnroutablePeer { peer } => { - event_string = "kad_event::UnroutablePeer"; - trace!(peer_id = %peer, "kad::Event: UnroutablePeer"); - } - kad::Event::RoutablePeer { peer, .. } => { - // We get this when we don't add a peer via the identify step. - // And we don't want to add these as they were rejected by identify for some reason. - event_string = "kad_event::RoutablePeer"; - trace!(peer_id = %peer, "kad::Event: RoutablePeer"); - } - other => { - event_string = "kad_event::Other"; - trace!("kad::Event ignored: {other:?}"); - } - } - - self.log_handling(event_string.to_string(), start.elapsed()); - - trace!( - "kad::Event handled in {:?}: {event_string:?}", - start.elapsed() - ); - - Ok(()) - } - - /// Check for changes in our close group - /// - pub(crate) fn check_for_change_in_our_close_group(&mut self) -> bool { - // this includes self - let closest_k_peers = self.get_closest_k_value_local_peers(); - - let new_closest_peers: Vec<_> = - closest_k_peers.into_iter().take(CLOSE_GROUP_SIZE).collect(); - - let old = self.close_group.iter().cloned().collect::>(); - let new_members: Vec<_> = new_closest_peers - .iter() - .filter(|p| !old.contains(p)) - .collect(); - if !new_members.is_empty() { - debug!("The close group has been updated. The new members are {new_members:?}"); - debug!("New close group: {new_closest_peers:?}"); - self.close_group = new_closest_peers; - true - } else { - false - } - } - - pub(crate) fn log_kbuckets(&mut self, peer: &PeerId) { - let distance = NetworkAddress::from_peer(self.self_peer_id) - .distance(&NetworkAddress::from_peer(*peer)); - info!("Peer {peer:?} has a {:?} distance to us", distance.ilog2()); - let mut kbucket_table_stats = vec![]; - let mut index = 0; - let mut total_peers = 0; - for kbucket in self.swarm.behaviour_mut().kademlia.kbuckets() { - let range = kbucket.range(); - total_peers += kbucket.num_entries(); - if let Some(distance) = range.0.ilog2() { - kbucket_table_stats.push((index, kbucket.num_entries(), distance)); - } else { - // This shall never happen. - error!("bucket #{index:?} is ourself ???!!!"); - } - index += 1; - } - info!("kBucketTable has {index:?} kbuckets {total_peers:?} peers, {kbucket_table_stats:?}"); - } - - // if target bucket is full, remove a bootstrap node if presents. - fn remove_bootstrap_from_full(&mut self, peer_id: PeerId) { - let mut shall_removed = None; - - if let Some(kbucket) = self.swarm.behaviour_mut().kademlia.kbucket(peer_id) { - if kbucket.num_entries() >= K_VALUE.into() { - if let Some(peers) = self.bootstrap_peers.get(&kbucket.range().0.ilog2()) { - for peer_entry in kbucket.iter() { - if peers.contains(peer_entry.node.key.preimage()) { - shall_removed = Some(*peer_entry.node.key.preimage()); - break; - } - } - } - } - } - if let Some(to_be_removed_bootstrap) = shall_removed { - trace!("Bootstrap node {to_be_removed_bootstrap:?} to be replaced by peer {peer_id:?}"); - let _entry = self - .swarm - .behaviour_mut() - .kademlia - .remove_peer(&to_be_removed_bootstrap); - } - } - - // Remove outdated connection to a peer if it is not in the RT. - fn remove_outdated_connections(&mut self) { - let mut shall_removed = vec![]; - - let timed_out_connections = - self.live_connected_peers - .iter() - .filter_map(|(connection_id, (peer_id, timeout))| { - if Instant::now() > *timeout { - Some((connection_id, peer_id)) - } else { - None - } - }); - - for (connection_id, peer_id) in timed_out_connections { - // Skip if the peer is present in our RT - if let Some(kbucket) = self.swarm.behaviour_mut().kademlia.kbucket(*peer_id) { - if kbucket - .iter() - .any(|peer_entry| *peer_id == *peer_entry.node.key.preimage()) - { - continue; - } - } - - // skip if the peer is a relay server that we're connected to - if self.relay_manager.keep_alive_peer(peer_id) { - continue; - } - - shall_removed.push((*connection_id, *peer_id)); - } - - if !shall_removed.is_empty() { - trace!( - "Current libp2p peers pool stats is {:?}", - self.swarm.network_info() - ); - trace!( - "Removing {} outdated live connections, still have {} left.", - shall_removed.len(), - self.live_connected_peers.len() - ); - trace!(?self.relay_manager); - - for (connection_id, peer_id) in shall_removed { - let _ = self.live_connected_peers.remove(&connection_id); - let result = self.swarm.close_connection(connection_id); - trace!("Removed outdated connection {connection_id:?} to {peer_id:?} with result: {result:?}"); - } - } - } - - fn add_keys_to_replication_fetcher( - &mut self, - sender: NetworkAddress, - incoming_keys: Vec<(NetworkAddress, RecordType)>, - ) { - let holder = if let Some(peer_id) = sender.as_peer_id() { - peer_id - } else { - warn!("Replication list sender is not a peer_id {sender:?}"); - return; - }; - - trace!( - "Received replication list from {holder:?} of {} keys", - incoming_keys.len() - ); - - // accept replication requests from the K_VALUE peers away, - // giving us some margin for replication - let closest_k_peers = self.get_closest_k_value_local_peers(); - if !closest_k_peers.contains(&holder) || holder == self.self_peer_id { - trace!("Holder {holder:?} is self or not in replication range."); - return; - } - - // On receive a replication_list from a close_group peer, we undertake two tasks: - // 1, For those keys that we don't have: - // fetch them if close enough to us - // 2, For those keys that we have and supposed to be held by the sender as well: - // start chunk_proof check against a randomly selected chunk type record to the sender - - // For fetching, only handle those non-exist and in close range keys - let keys_to_store = - self.select_non_existent_records_for_replications(&incoming_keys, &closest_k_peers); - - if keys_to_store.is_empty() { - debug!("Empty keys to store after adding to"); - } else { - #[allow(clippy::mutable_key_type)] - let all_keys = self - .swarm - .behaviour_mut() - .kademlia - .store_mut() - .record_addresses_ref(); - let keys_to_fetch = self - .replication_fetcher - .add_keys(holder, keys_to_store, all_keys); - if keys_to_fetch.is_empty() { - trace!("no waiting keys to fetch from the network"); - } else { - self.send_event(NetworkEvent::KeysToFetchForReplication(keys_to_fetch)); - } - } - - // Only trigger chunk_proof check when received a periodical replication request. - if incoming_keys.len() > 1 { - let keys_to_verify = self.select_verification_data_candidates(sender); - - if keys_to_verify.is_empty() { - debug!("No valid candidate to be checked against peer {holder:?}"); - } else { - self.send_event(NetworkEvent::ChunkProofVerification { - peer_id: holder, - keys_to_verify, - }); - } - } - } - - /// Checks suggested records against what we hold, so we only - /// enqueue what we do not have - fn select_non_existent_records_for_replications( - &mut self, - incoming_keys: &[(NetworkAddress, RecordType)], - closest_k_peers: &Vec, - ) -> Vec<(NetworkAddress, RecordType)> { - #[allow(clippy::mutable_key_type)] - let locally_stored_keys = self - .swarm - .behaviour_mut() - .kademlia - .store_mut() - .record_addresses_ref(); - let non_existent_keys: Vec<_> = incoming_keys - .iter() - .filter(|(addr, record_type)| { - let key = addr.to_record_key(); - let local = locally_stored_keys.get(&key); - - // if we have a local value of matching record_type, we don't need to fetch it - if let Some((_, local_record_type)) = local { - let not_same_type = local_record_type != record_type; - if not_same_type { - // Shall only happens for Register - info!("Record {addr:?} has different type: local {local_record_type:?}, incoming {record_type:?}"); - } - not_same_type - } else { - true - } - }) - .collect(); - - non_existent_keys - .into_iter() - .filter_map(|(key, record_type)| { - if Self::is_in_close_range(&self.self_peer_id, key, closest_k_peers) { - Some((key.clone(), record_type.clone())) - } else { - // Reduce the log level as there will always be around 40% records being - // out of the close range, as the sender side is using `CLOSE_GROUP_SIZE + 2` - // to send our replication list to provide addressing margin. - // Given there will normally be 6 nodes sending such list with interval of 5-10s, - // this will accumulate to a lot of logs with the increasing records uploaded. - trace!("not in close range for key {key:?}"); - None - } - }) - .collect() - } - - // A close target doesn't falls into the close peers range: - // For example, a node b11111X has an RT: [(1, b1111), (2, b111), (5, b11), (9, b1), (7, b0)] - // Then for a target bearing b011111 as prefix, all nodes in (7, b0) are its close_group peers. - // Then the node b11111X. But b11111X's close_group peers [(1, b1111), (2, b111), (5, b11)] - // are none among target b011111's close range. - // Hence, the ilog2 calculation based on close_range cannot cover such case. - // And have to sort all nodes to figure out whether self is among the close_group to the target. - pub(crate) fn is_in_close_range( - our_peer_id: &PeerId, - target: &NetworkAddress, - all_peers: &Vec, - ) -> bool { - if all_peers.len() <= REPLICATION_PEERS_COUNT { - return true; - } - - // Margin of 2 to allow our RT being bit lagging. - match sort_peers_by_address(all_peers, target, REPLICATION_PEERS_COUNT) { - Ok(close_group) => close_group.contains(&our_peer_id), - Err(err) => { - warn!("Could not get sorted peers for {target:?} with error {err:?}"); - true - } - } - } - - /// Check among all chunk type records that we have, select those close to the peer, - /// and randomly pick one as the verification candidate. - #[allow(clippy::mutable_key_type)] - fn select_verification_data_candidates(&mut self, peer: NetworkAddress) -> Vec { - let mut closest_peers = self - .swarm - .behaviour_mut() - .kademlia - .get_closest_local_peers(&self.self_peer_id.into()) - .map(|peer| peer.into_preimage()) - .take(20) - .collect_vec(); - closest_peers.push(self.self_peer_id); - - let target_peer = if let Some(peer_id) = peer.as_peer_id() { - peer_id - } else { - error!("Target {peer:?} is not a valid PeerId"); - return vec![]; - }; - - #[allow(clippy::mutable_key_type)] - let all_keys = self - .swarm - .behaviour_mut() - .kademlia - .store_mut() - .record_addresses_ref(); - - // Targeted chunk type record shall be expected within the close range from our perspective. - let mut verify_candidates: Vec = all_keys - .values() - .filter_map(|(addr, record_type)| { - if RecordType::Chunk == *record_type { - match sort_peers_by_address(&closest_peers, addr, CLOSE_GROUP_SIZE) { - Ok(close_group) => { - if close_group.contains(&&target_peer) { - Some(addr.clone()) - } else { - None - } - } - Err(err) => { - warn!("Could not get sorted peers for {addr:?} with error {err:?}"); - None - } - } - } else { - None - } - }) - .collect(); - - verify_candidates.sort_by_key(|a| peer.distance(a)); - - // To ensure the candidate mush have to be held by the peer, - // we only carry out check when there are already certain amount of chunks uploaded - // AND choose candidate from certain reduced range. - if verify_candidates.len() > 50 { - let index: usize = OsRng.gen_range(0..(verify_candidates.len() / 2)); - vec![verify_candidates[index].clone()] - } else { - vec![] - } - } -} - -/// Helper function to print formatted connection role info. -fn endpoint_str(endpoint: &libp2p::core::ConnectedPoint) -> String { - match endpoint { - libp2p::core::ConnectedPoint::Dialer { address, .. } => { - format!("outgoing ({address})") - } - libp2p::core::ConnectedPoint::Listener { send_back_addr, .. } => { - format!("incoming ({send_back_addr})") - } - } -} diff --git a/sn_networking/src/event/kad.rs b/sn_networking/src/event/kad.rs new file mode 100644 index 0000000000..7f3e111751 --- /dev/null +++ b/sn_networking/src/event/kad.rs @@ -0,0 +1,585 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::{ + driver::PendingGetClosestType, get_quorum_value, GetRecordCfg, GetRecordError, NetworkError, + NetworkEvent, Result, SwarmDriver, CLOSE_GROUP_SIZE, +}; +use itertools::Itertools; +use libp2p::kad::{ + self, GetClosestPeersError, InboundRequest, PeerRecord, ProgressStep, QueryId, QueryResult, + QueryStats, Record, K_VALUE, +}; +use sn_protocol::PrettyPrintRecordKey; +use std::{ + collections::{hash_map::Entry, HashSet}, + time::Instant, +}; +use tokio::sync::oneshot; +use xor_name::XorName; + +impl SwarmDriver { + pub(super) fn handle_kad_event(&mut self, kad_event: libp2p::kad::Event) -> Result<()> { + let start = Instant::now(); + let event_string; + + match kad_event { + kad::Event::OutboundQueryProgressed { + id, + result: QueryResult::GetClosestPeers(Ok(ref closest_peers)), + ref stats, + ref step, + } => { + event_string = "kad_event::get_closest_peers"; + trace!( + "Query task {id:?} of key {:?} returned with peers {:?}, {stats:?} - {step:?}", + hex::encode(closest_peers.key.clone()), + closest_peers.peers, + ); + + if let Entry::Occupied(mut entry) = self.pending_get_closest_peers.entry(id) { + let (_, current_closest) = entry.get_mut(); + + // TODO: consider order the result and terminate when reach any of the + // following criteria: + // 1, `stats.num_pending()` is 0 + // 2, `stats.duration()` is longer than a defined period + current_closest.extend(closest_peers.peers.clone()); + if current_closest.len() >= usize::from(K_VALUE) || step.last { + let (get_closest_type, current_closest) = entry.remove(); + match get_closest_type { + PendingGetClosestType::NetworkDiscovery => self + .network_discovery + .handle_get_closest_query(current_closest), + PendingGetClosestType::FunctionCall(sender) => { + sender + .send(current_closest) + .map_err(|_| NetworkError::InternalMsgChannelDropped)?; + } + } + } + } else { + trace!("Can't locate query task {id:?}, it has likely been completed already."); + return Err(NetworkError::ReceivedKademliaEventDropped { + query_id: id, + event: "GetClosestPeers Ok".to_string(), + }); + } + } + // Handle GetClosestPeers timeouts + kad::Event::OutboundQueryProgressed { + id, + result: QueryResult::GetClosestPeers(Err(ref err)), + ref stats, + ref step, + } => { + event_string = "kad_event::get_closest_peers_err"; + error!("GetClosest Query task {id:?} errored with {err:?}, {stats:?} - {step:?}"); + + let (get_closest_type, mut current_closest) = + self.pending_get_closest_peers.remove(&id).ok_or_else(|| { + trace!( + "Can't locate query task {id:?}, it has likely been completed already." + ); + NetworkError::ReceivedKademliaEventDropped { + query_id: id, + event: "Get ClosestPeers error".to_string(), + } + })?; + + // We have `current_closest` from previous progress, + // and `peers` from `GetClosestPeersError`. + // Trust them and leave for the caller to check whether they are enough. + match err { + GetClosestPeersError::Timeout { ref peers, .. } => { + current_closest.extend(peers); + } + } + + match get_closest_type { + PendingGetClosestType::NetworkDiscovery => self + .network_discovery + .handle_get_closest_query(current_closest), + PendingGetClosestType::FunctionCall(sender) => { + sender + .send(current_closest) + .map_err(|_| NetworkError::InternalMsgChannelDropped)?; + } + } + } + + kad::Event::OutboundQueryProgressed { + id, + result: QueryResult::GetRecord(Ok(kad::GetRecordOk::FoundRecord(peer_record))), + stats, + step, + } => { + event_string = "kad_event::get_record::found"; + trace!( + "Query task {id:?} returned with record {:?} from peer {:?}, {stats:?} - {step:?}", + PrettyPrintRecordKey::from(&peer_record.record.key), + peer_record.peer + ); + self.accumulate_get_record_found(id, peer_record, stats, step)?; + } + kad::Event::OutboundQueryProgressed { + id, + result: + QueryResult::GetRecord(Ok(kad::GetRecordOk::FinishedWithNoAdditionalRecord { + cache_candidates, + })), + stats, + step, + } => { + event_string = "kad_event::get_record::finished_no_additional"; + trace!("Query task {id:?} of get_record completed with {stats:?} - {step:?} - {cache_candidates:?}"); + self.handle_get_record_finished(id, step)?; + } + kad::Event::OutboundQueryProgressed { + id, + result: QueryResult::GetRecord(Err(get_record_err)), + stats, + step, + } => { + // log the errors + match &get_record_err { + kad::GetRecordError::NotFound { key, closest_peers } => { + event_string = "kad_event::GetRecordError::NotFound"; + info!("Query task {id:?} NotFound record {:?} among peers {closest_peers:?}, {stats:?} - {step:?}", + PrettyPrintRecordKey::from(key)); + } + kad::GetRecordError::QuorumFailed { + key, + records, + quorum, + } => { + event_string = "kad_event::GetRecordError::QuorumFailed"; + let pretty_key = PrettyPrintRecordKey::from(key); + let peers = records + .iter() + .map(|peer_record| peer_record.peer) + .collect_vec(); + info!("Query task {id:?} QuorumFailed record {pretty_key:?} among peers {peers:?} with quorum {quorum:?}, {stats:?} - {step:?}"); + } + kad::GetRecordError::Timeout { key } => { + event_string = "kad_event::GetRecordError::Timeout"; + let pretty_key = PrettyPrintRecordKey::from(key); + + debug!( + "Query task {id:?} timed out when looking for record {pretty_key:?}" + ); + } + } + self.handle_get_record_error(id, get_record_err, stats, step)?; + } + kad::Event::OutboundQueryProgressed { + id, + result: QueryResult::PutRecord(Err(put_record_err)), + stats, + step, + } => { + // Currently, only `client` calls `put_record_to` to upload data. + // The result of such operation is not critical to client in general. + // However, if client keeps receiving error responses, it may indicating: + // 1, Client itself is with slow connection + // OR + // 2, The payee node selected could be in trouble + // + // TODO: Figure out which payee node the error response is related to, + // and may exclude that node from later on payee selection. + let (key, success, quorum) = match &put_record_err { + kad::PutRecordError::QuorumFailed { + key, + success, + quorum, + } => { + event_string = "kad_event::PutRecordError::QuorumFailed"; + (key, success, quorum) + } + kad::PutRecordError::Timeout { + key, + success, + quorum, + } => { + event_string = "kad_event::PutRecordError::Timeout"; + (key, success, quorum) + } + }; + error!("Query task {id:?} failed put record {:?} {:?}, required quorum {quorum}, stored on {success:?}, {stats:?} - {step:?}", + PrettyPrintRecordKey::from(key), event_string); + } + kad::Event::OutboundQueryProgressed { + id, + result: QueryResult::PutRecord(Ok(put_record_ok)), + stats, + step, + } => { + event_string = "kad_event::PutRecordOk"; + trace!( + "Query task {id:?} put record {:?} ok, {stats:?} - {step:?}", + PrettyPrintRecordKey::from(&put_record_ok.key) + ); + } + // Shall no longer receive this event + kad::Event::OutboundQueryProgressed { + id, + result: QueryResult::Bootstrap(bootstrap_result), + step, + .. + } => { + event_string = "kad_event::OutboundQueryProgressed::Bootstrap"; + // here BootstrapOk::num_remaining refers to the remaining random peer IDs to query, one per + // bucket that still needs refreshing. + trace!("Kademlia Bootstrap with {id:?} progressed with {bootstrap_result:?} and step {step:?}"); + } + kad::Event::RoutingUpdated { + peer, + is_new_peer, + old_peer, + .. + } => { + event_string = "kad_event::RoutingUpdated"; + if is_new_peer { + self.connected_peers = self.connected_peers.saturating_add(1); + + info!("New peer added to routing table: {peer:?}, now we have #{} connected peers", self.connected_peers); + self.log_kbuckets(&peer); + + // This should only happen once + if self.bootstrap.notify_new_peer() { + info!("Performing the first bootstrap"); + self.trigger_network_discovery(); + } + self.send_event(NetworkEvent::PeerAdded(peer, self.connected_peers)); + } + + info!("kad_event::RoutingUpdated {:?}: {peer:?}, is_new_peer: {is_new_peer:?} old_peer: {old_peer:?}", self.connected_peers); + if old_peer.is_some() { + self.connected_peers = self.connected_peers.saturating_sub(1); + + info!("Evicted old peer on new peer join: {old_peer:?}"); + self.send_event(NetworkEvent::PeerRemoved(peer, self.connected_peers)); + self.log_kbuckets(&peer); + } + let _ = self.check_for_change_in_our_close_group(); + } + kad::Event::InboundRequest { + request: InboundRequest::PutRecord { .. }, + } => { + event_string = "kad_event::InboundRequest::PutRecord"; + // Ignored to reduce logging. When `Record filtering` is enabled, + // the `record` variable will contain the content for further validation before put. + } + kad::Event::InboundRequest { + request: InboundRequest::FindNode { .. }, + } => { + event_string = "kad_event::InboundRequest::FindNode"; + // Ignored to reduce logging. With continuous bootstrap, this is triggered often. + } + kad::Event::InboundRequest { + request: + InboundRequest::GetRecord { + num_closer_peers, + present_locally, + }, + } => { + event_string = "kad_event::InboundRequest::GetRecord"; + if !present_locally && num_closer_peers < CLOSE_GROUP_SIZE { + trace!("InboundRequest::GetRecord doesn't have local record, with {num_closer_peers:?} closer_peers"); + } + } + kad::Event::UnroutablePeer { peer } => { + event_string = "kad_event::UnroutablePeer"; + trace!(peer_id = %peer, "kad::Event: UnroutablePeer"); + } + kad::Event::RoutablePeer { peer, .. } => { + // We get this when we don't add a peer via the identify step. + // And we don't want to add these as they were rejected by identify for some reason. + event_string = "kad_event::RoutablePeer"; + trace!(peer_id = %peer, "kad::Event: RoutablePeer"); + } + other => { + event_string = "kad_event::Other"; + trace!("kad::Event ignored: {other:?}"); + } + } + + self.log_handling(event_string.to_string(), start.elapsed()); + + trace!( + "kad::Event handled in {:?}: {event_string:?}", + start.elapsed() + ); + + Ok(()) + } + + // For `get_record` returning behaviour: + // 1, targeting a non-existing entry + // there will only be one event of `kad::Event::OutboundQueryProgressed` + // with `ProgressStep::last` to be `true` + // `QueryStats::requests` to be 20 (K-Value) + // `QueryStats::success` to be over majority of the requests + // `err::NotFound::closest_peers` contains a list of CLOSE_GROUP_SIZE peers + // 2, targeting an existing entry + // there will a sequence of (at least CLOSE_GROUP_SIZE) events of + // `kad::Event::OutboundQueryProgressed` to be received + // with `QueryStats::end` always being `None` + // `ProgressStep::last` all to be `false` + // `ProgressStep::count` to be increased with step of 1 + // capped and stopped at CLOSE_GROUP_SIZE, may have duplicated counts + // `PeerRecord::peer` could be None to indicate from self + // in which case it always use a duplicated `ProgressStep::count` + // the sequence will be completed with `FinishedWithNoAdditionalRecord` + // where: `cache_candidates`: being the peers supposed to hold the record but not + // `ProgressStep::count`: to be `number of received copies plus one` + // `ProgressStep::last` to be `true` + + /// Accumulates the GetRecord query results + /// If we get enough responses (quorum) for a record with the same content hash: + /// - we return the Record after comparing with the target record. This might return RecordDoesNotMatch if the + /// check fails. + /// - if multiple content hashes are found, we return a SplitRecord Error + /// And then we stop the kad query as we are done here. + fn accumulate_get_record_found( + &mut self, + query_id: QueryId, + peer_record: PeerRecord, + _stats: QueryStats, + step: ProgressStep, + ) -> Result<()> { + let peer_id = if let Some(peer_id) = peer_record.peer { + peer_id + } else { + self.self_peer_id + }; + let pretty_key = PrettyPrintRecordKey::from(&peer_record.record.key).into_owned(); + + if let Entry::Occupied(mut entry) = self.pending_get_record.entry(query_id) { + let (_sender, result_map, cfg) = entry.get_mut(); + + if !cfg.expected_holders.is_empty() { + if cfg.expected_holders.remove(&peer_id) { + debug!("For record {pretty_key:?} task {query_id:?}, received a copy from an expected holder {peer_id:?}"); + } else { + debug!("For record {pretty_key:?} task {query_id:?}, received a copy from an unexpected holder {peer_id:?}"); + } + } + + // Insert the record and the peer into the result_map. + let record_content_hash = XorName::from_content(&peer_record.record.value); + let responded_peers = + if let Entry::Occupied(mut entry) = result_map.entry(record_content_hash) { + let (_, peer_list) = entry.get_mut(); + let _ = peer_list.insert(peer_id); + peer_list.len() + } else { + let mut peer_list = HashSet::new(); + let _ = peer_list.insert(peer_id); + result_map.insert(record_content_hash, (peer_record.record.clone(), peer_list)); + 1 + }; + + let expected_answers = get_quorum_value(&cfg.get_quorum); + + trace!("Expecting {expected_answers:?} answers for record {pretty_key:?} task {query_id:?}, received {responded_peers} so far"); + + if responded_peers >= expected_answers { + if !cfg.expected_holders.is_empty() { + debug!("For record {pretty_key:?} task {query_id:?}, fetch completed with non-responded expected holders {:?}", cfg.expected_holders); + } + let cfg = cfg.clone(); + + // Remove the query task and consume the variables. + let (sender, result_map, _) = entry.remove(); + + if result_map.len() == 1 { + Self::send_record_after_checking_target(sender, peer_record.record, &cfg)?; + } else { + debug!("For record {pretty_key:?} task {query_id:?}, fetch completed with split record"); + sender + .send(Err(GetRecordError::SplitRecord { result_map })) + .map_err(|_| NetworkError::InternalMsgChannelDropped)?; + } + + // Stop the query; possibly stops more nodes from being queried. + if let Some(mut query) = self.swarm.behaviour_mut().kademlia.query_mut(&query_id) { + query.finish(); + } + } else if usize::from(step.count) >= CLOSE_GROUP_SIZE { + debug!("For record {pretty_key:?} task {query_id:?}, got {:?} with {} versions so far.", + step.count, result_map.len()); + } + } else { + // return error if the entry cannot be found + return Err(NetworkError::ReceivedKademliaEventDropped { + query_id, + event: format!("Accumulate Get Record of {pretty_key:?}"), + }); + } + Ok(()) + } + + /// Handles the possible cases when a GetRecord Query completes. + /// The accumulate_get_record_found returns the record if the quorum is satisfied, but, if we have reached this point + /// then we did not get enough records or we got split records (which prevented the quorum to pass). + /// Returns the following errors: + /// RecordNotFound if the result_map is empty. + /// NotEnoughCopies if there is only a single content hash version. + /// SplitRecord if there are multiple content hash versions. + fn handle_get_record_finished(&mut self, query_id: QueryId, step: ProgressStep) -> Result<()> { + // return error if the entry cannot be found + if let Some((sender, result_map, cfg)) = self.pending_get_record.remove(&query_id) { + let num_of_versions = result_map.len(); + let (result, log_string) = if let Some((record, from_peers)) = + result_map.values().next() + { + let result = if num_of_versions == 1 { + Err(GetRecordError::NotEnoughCopies { + record: record.clone(), + expected: get_quorum_value(&cfg.get_quorum), + got: from_peers.len(), + }) + } else { + Err(GetRecordError::SplitRecord { + result_map: result_map.clone(), + }) + }; + + ( + result, + format!("Getting record {:?} completed with only {:?} copies received, and {num_of_versions} versions.", + PrettyPrintRecordKey::from(&record.key), usize::from(step.count) - 1) + ) + } else { + ( + Err(GetRecordError::RecordNotFound), + format!("Getting record task {query_id:?} completed with step count {:?}, but no copy found.", step.count), + ) + }; + + if cfg.expected_holders.is_empty() { + debug!("{log_string}"); + } else { + debug!( + "{log_string}, and {:?} expected holders not responded", + cfg.expected_holders + ); + } + + sender + .send(result) + .map_err(|_| NetworkError::InternalMsgChannelDropped)?; + } else { + // We manually perform `query.finish()` if we return early from accumulate fn. + // Thus we will still get FinishedWithNoAdditionalRecord. + trace!("Can't locate query task {query_id:?} during GetRecord finished. We might have already returned the result to the sender."); + } + Ok(()) + } + + /// Handles the possible cases when a kad GetRecord returns an error. + /// If we get NotFound/QuorumFailed, we return a RecordNotFound error. Kad currently does not enforce any quorum. + /// If we get a Timeout: + /// - return a QueryTimeout if we get a split record (?) if we have multiple content hashes. + /// - if the quorum is satisfied, we return the record after comparing it with the target record. This might return + /// RecordDoesNotMatch if the check fails. + /// - else we return q QueryTimeout error. + fn handle_get_record_error( + &mut self, + query_id: QueryId, + get_record_err: kad::GetRecordError, + _stats: QueryStats, + _step: ProgressStep, + ) -> Result<()> { + match &get_record_err { + kad::GetRecordError::NotFound { .. } | kad::GetRecordError::QuorumFailed { .. } => { + // return error if the entry cannot be found + let (sender, _, cfg) = + self.pending_get_record.remove(&query_id).ok_or_else(|| { + trace!("Can't locate query task {query_id:?}, it has likely been completed already."); + NetworkError::ReceivedKademliaEventDropped { + query_id, + event: "GetRecordError NotFound or QuorumFailed".to_string(), + } + })?; + + if cfg.expected_holders.is_empty() { + info!("Get record task {query_id:?} failed with error {get_record_err:?}"); + } else { + debug!("Get record task {query_id:?} failed with {:?} expected holders not responded, error {get_record_err:?}", cfg.expected_holders); + } + sender + .send(Err(GetRecordError::RecordNotFound)) + .map_err(|_| NetworkError::InternalMsgChannelDropped)?; + } + kad::GetRecordError::Timeout { key } => { + // return error if the entry cannot be found + let pretty_key = PrettyPrintRecordKey::from(key); + let (sender, result_map, cfg) = + self.pending_get_record.remove(&query_id).ok_or_else(|| { + trace!( + "Can't locate query task {query_id:?} for {pretty_key:?}, it has likely been completed already." + ); + NetworkError::ReceivedKademliaEventDropped { + query_id, + event: format!("GetRecordError Timeout {pretty_key:?}"), + } + })?; + + let required_response_count = get_quorum_value(&cfg.get_quorum); + + // if we've a split over the result xorname, then we don't attempt to resolve this here. + // Retry and resolve through normal flows without a timeout. + // todo: is the above still the case? Why don't we return a split record error. + if result_map.len() > 1 { + warn!( + "Get record task {query_id:?} for {pretty_key:?} timed out with split result map" + ); + sender + .send(Err(GetRecordError::QueryTimeout)) + .map_err(|_| NetworkError::InternalMsgChannelDropped)?; + + return Ok(()); + } + + // if we have enough responses here, we can return the record + if let Some((record, peers)) = result_map.values().next() { + if peers.len() >= required_response_count { + Self::send_record_after_checking_target(sender, record.clone(), &cfg)?; + return Ok(()); + } + } + + warn!("Get record task {query_id:?} for {pretty_key:?} returned insufficient responses. {:?} did not return record", cfg.expected_holders); + // Otherwise report the timeout + sender + .send(Err(GetRecordError::QueryTimeout)) + .map_err(|_| NetworkError::InternalMsgChannelDropped)?; + } + } + + Ok(()) + } + + fn send_record_after_checking_target( + sender: oneshot::Sender>, + record: Record, + cfg: &GetRecordCfg, + ) -> Result<()> { + if cfg.target_record.is_none() || cfg.does_target_match(&record) { + sender + .send(Ok(record)) + .map_err(|_| NetworkError::InternalMsgChannelDropped) + } else { + sender + .send(Err(GetRecordError::RecordDoesNotMatch(record))) + .map_err(|_| NetworkError::InternalMsgChannelDropped) + } + } +} diff --git a/sn_networking/src/event/mod.rs b/sn_networking/src/event/mod.rs new file mode 100644 index 0000000000..e4112d1549 --- /dev/null +++ b/sn_networking/src/event/mod.rs @@ -0,0 +1,273 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +mod kad; +mod request_response; +mod swarm; + +use crate::{driver::SwarmDriver, error::Result, CLOSE_GROUP_SIZE}; +use core::fmt; +use custom_debug::Debug as CustomDebug; +#[cfg(feature = "local-discovery")] +use libp2p::mdns; +use libp2p::{ + kad::{Record, RecordKey}, + request_response::ResponseChannel as PeerResponseChannel, + Multiaddr, PeerId, +}; + +use sn_protocol::{ + messages::{Query, Request, Response}, + NetworkAddress, PrettyPrintRecordKey, +}; +use sn_transfers::PaymentQuote; +use std::{ + collections::{BTreeSet, HashSet}, + fmt::{Debug, Formatter}, +}; +use tokio::sync::oneshot; + +/// NodeEvent enum +#[derive(CustomDebug)] +pub(super) enum NodeEvent { + MsgReceived(libp2p::request_response::Event), + Kademlia(libp2p::kad::Event), + #[cfg(feature = "local-discovery")] + Mdns(Box), + Identify(Box), + Dcutr(Box), + RelayClient(Box), + RelayServer(Box), +} + +impl From> for NodeEvent { + fn from(event: libp2p::request_response::Event) -> Self { + NodeEvent::MsgReceived(event) + } +} + +impl From for NodeEvent { + fn from(event: libp2p::kad::Event) -> Self { + NodeEvent::Kademlia(event) + } +} + +#[cfg(feature = "local-discovery")] +impl From for NodeEvent { + fn from(event: mdns::Event) -> Self { + NodeEvent::Mdns(Box::new(event)) + } +} + +impl From for NodeEvent { + fn from(event: libp2p::identify::Event) -> Self { + NodeEvent::Identify(Box::new(event)) + } +} +impl From for NodeEvent { + fn from(event: libp2p::dcutr::Event) -> Self { + NodeEvent::Dcutr(Box::new(event)) + } +} +impl From for NodeEvent { + fn from(event: libp2p::relay::client::Event) -> Self { + NodeEvent::RelayClient(Box::new(event)) + } +} +impl From for NodeEvent { + fn from(event: libp2p::relay::Event) -> Self { + NodeEvent::RelayServer(Box::new(event)) + } +} + +#[derive(CustomDebug)] +/// Channel to send the `Response` through. +pub enum MsgResponder { + /// Respond to a request from `self` through a simple one-shot channel. + FromSelf(Option>>), + /// Respond to a request from a peer in the network. + FromPeer(PeerResponseChannel), +} + +#[allow(clippy::large_enum_variant)] +/// Events forwarded by the underlying Network; to be used by the upper layers +pub enum NetworkEvent { + /// Incoming `Query` from a peer + QueryRequestReceived { + /// Query + query: Query, + /// The channel to send the `Response` through + channel: MsgResponder, + }, + /// Handles the responses that are not awaited at the call site + ResponseReceived { + /// Response + res: Response, + }, + /// Peer has been added to the Routing Table. And the number of connected peers. + PeerAdded(PeerId, usize), + /// Peer has been removed from the Routing Table. And the number of connected peers. + PeerRemoved(PeerId, usize), + /// The peer does not support our protocol + PeerWithUnsupportedProtocol { + our_protocol: String, + their_protocol: String, + }, + /// The peer is now considered as a bad node, due to the detected bad behaviour + PeerConsideredAsBad { + detected_by: PeerId, + bad_peer: PeerId, + bad_behaviour: String, + }, + /// The records bearing these keys are to be fetched from the holder or the network + KeysToFetchForReplication(Vec<(PeerId, RecordKey)>), + /// Started listening on a new address + NewListenAddr(Multiaddr), + /// Report unverified record + UnverifiedRecord(Record), + /// Terminate Node on unrecoverable errors + TerminateNode { reason: TerminateNodeReason }, + /// List of peer nodes that failed to fetch replication copy from. + FailedToFetchHolders(BTreeSet), + /// A peer in RT that supposed to be verified. + BadNodeVerification { peer_id: PeerId }, + /// Quotes to be verified + QuoteVerification { quotes: Vec<(PeerId, PaymentQuote)> }, + /// Carry out chunk proof check against the specified record and peer + ChunkProofVerification { + peer_id: PeerId, + keys_to_verify: Vec, + }, +} + +/// Terminate node for the following reason +#[derive(Debug, Clone)] +pub enum TerminateNodeReason { + HardDiskWriteError, +} + +// Manually implement Debug as `#[debug(with = "unverified_record_fmt")]` not working as expected. +impl Debug for NetworkEvent { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + NetworkEvent::QueryRequestReceived { query, .. } => { + write!(f, "NetworkEvent::QueryRequestReceived({query:?})") + } + NetworkEvent::ResponseReceived { res, .. } => { + write!(f, "NetworkEvent::ResponseReceived({res:?})") + } + NetworkEvent::PeerAdded(peer_id, connected_peers) => { + write!(f, "NetworkEvent::PeerAdded({peer_id:?}, {connected_peers})") + } + NetworkEvent::PeerRemoved(peer_id, connected_peers) => { + write!( + f, + "NetworkEvent::PeerRemoved({peer_id:?}, {connected_peers})" + ) + } + NetworkEvent::PeerWithUnsupportedProtocol { + our_protocol, + their_protocol, + } => { + write!(f, "NetworkEvent::PeerWithUnsupportedProtocol({our_protocol:?}, {their_protocol:?})") + } + NetworkEvent::PeerConsideredAsBad { + bad_peer, + bad_behaviour, + .. + } => { + write!( + f, + "NetworkEvent::PeerConsideredAsBad({bad_peer:?}, {bad_behaviour:?})" + ) + } + NetworkEvent::KeysToFetchForReplication(list) => { + let keys_len = list.len(); + write!(f, "NetworkEvent::KeysForReplication({keys_len:?})") + } + NetworkEvent::NewListenAddr(addr) => { + write!(f, "NetworkEvent::NewListenAddr({addr:?})") + } + NetworkEvent::UnverifiedRecord(record) => { + let pretty_key = PrettyPrintRecordKey::from(&record.key); + write!(f, "NetworkEvent::UnverifiedRecord({pretty_key:?})") + } + NetworkEvent::TerminateNode { reason } => { + write!(f, "NetworkEvent::TerminateNode({reason:?})") + } + NetworkEvent::FailedToFetchHolders(bad_nodes) => { + write!(f, "NetworkEvent::FailedToFetchHolders({bad_nodes:?})") + } + NetworkEvent::BadNodeVerification { peer_id } => { + write!(f, "NetworkEvent::BadNodeVerification({peer_id:?})") + } + NetworkEvent::QuoteVerification { quotes } => { + write!( + f, + "NetworkEvent::QuoteVerification({} quotes)", + quotes.len() + ) + } + NetworkEvent::ChunkProofVerification { + peer_id, + keys_to_verify, + } => { + write!( + f, + "NetworkEvent::ChunkProofVerification({peer_id:?} {keys_to_verify:?})" + ) + } + } + } +} + +impl SwarmDriver { + /// Check for changes in our close group + pub(crate) fn check_for_change_in_our_close_group(&mut self) -> bool { + // this includes self + let closest_k_peers = self.get_closest_k_value_local_peers(); + + let new_closest_peers: Vec<_> = + closest_k_peers.into_iter().take(CLOSE_GROUP_SIZE).collect(); + + let old = self.close_group.iter().cloned().collect::>(); + let new_members: Vec<_> = new_closest_peers + .iter() + .filter(|p| !old.contains(p)) + .collect(); + if !new_members.is_empty() { + debug!("The close group has been updated. The new members are {new_members:?}"); + debug!("New close group: {new_closest_peers:?}"); + self.close_group = new_closest_peers; + true + } else { + false + } + } + + pub(crate) fn log_kbuckets(&mut self, peer: &PeerId) { + let distance = NetworkAddress::from_peer(self.self_peer_id) + .distance(&NetworkAddress::from_peer(*peer)); + info!("Peer {peer:?} has a {:?} distance to us", distance.ilog2()); + let mut kbucket_table_stats = vec![]; + let mut index = 0; + let mut total_peers = 0; + for kbucket in self.swarm.behaviour_mut().kademlia.kbuckets() { + let range = kbucket.range(); + total_peers += kbucket.num_entries(); + if let Some(distance) = range.0.ilog2() { + kbucket_table_stats.push((index, kbucket.num_entries(), distance)); + } else { + // This shall never happen. + error!("bucket #{index:?} is ourself ???!!!"); + } + index += 1; + } + info!("kBucketTable has {index:?} kbuckets {total_peers:?} peers, {kbucket_table_stats:?}"); + } +} diff --git a/sn_networking/src/event/request_response.rs b/sn_networking/src/event/request_response.rs new file mode 100644 index 0000000000..e0f102a9d1 --- /dev/null +++ b/sn_networking/src/event/request_response.rs @@ -0,0 +1,394 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::{ + sort_peers_by_address, MsgResponder, NetworkError, NetworkEvent, SwarmDriver, CLOSE_GROUP_SIZE, + REPLICATION_PEERS_COUNT, +}; +use itertools::Itertools; +use libp2p::{ + request_response::{self, Message}, + PeerId, +}; +use rand::{rngs::OsRng, Rng}; +use sn_protocol::{ + messages::{CmdResponse, Request, Response}, + storage::RecordType, + NetworkAddress, +}; + +impl SwarmDriver { + /// Forwards `Request` to the upper layers using `Sender`. Sends `Response` to the peers + pub(super) fn handle_req_resp_events( + &mut self, + event: request_response::Event, + ) -> Result<(), NetworkError> { + match event { + request_response::Event::Message { message, peer } => match message { + Message::Request { + request, + channel, + request_id, + .. + } => { + trace!("Received request {request_id:?} from peer {peer:?}, req: {request:?}"); + // If the request is replication or quote verification, + // we can handle it and send the OK response here. + // As the handle result is unimportant to the sender. + match request { + Request::Cmd(sn_protocol::messages::Cmd::Replicate { holder, keys }) => { + let response = Response::Cmd( + sn_protocol::messages::CmdResponse::Replicate(Ok(())), + ); + self.swarm + .behaviour_mut() + .request_response + .send_response(channel, response) + .map_err(|_| NetworkError::InternalMsgChannelDropped)?; + + self.add_keys_to_replication_fetcher(holder, keys); + } + Request::Cmd(sn_protocol::messages::Cmd::QuoteVerification { + quotes, + .. + }) => { + let response = Response::Cmd( + sn_protocol::messages::CmdResponse::QuoteVerification(Ok(())), + ); + self.swarm + .behaviour_mut() + .request_response + .send_response(channel, response) + .map_err(|_| NetworkError::InternalMsgChannelDropped)?; + + // The keypair is required to verify the quotes, + // hence throw it up to Network layer for further actions. + let quotes = quotes + .iter() + .filter_map(|(peer_address, quote)| { + peer_address + .as_peer_id() + .map(|peer_id| (peer_id, quote.clone())) + }) + .collect(); + self.send_event(NetworkEvent::QuoteVerification { quotes }) + } + Request::Cmd(sn_protocol::messages::Cmd::PeerConsideredAsBad { + detected_by, + bad_peer, + bad_behaviour, + }) => { + let response = Response::Cmd( + sn_protocol::messages::CmdResponse::PeerConsideredAsBad(Ok(())), + ); + self.swarm + .behaviour_mut() + .request_response + .send_response(channel, response) + .map_err(|_| NetworkError::InternalMsgChannelDropped)?; + + if bad_peer == NetworkAddress::from_peer(self.self_peer_id) { + warn!("Peer {detected_by:?} consider us as BAD, due to {bad_behaviour:?}."); + // TODO: shall we terminate self after received such notifications + // from the majority close_group nodes around us? + } else { + error!("Received a bad_peer notification from {detected_by:?}, targeting {bad_peer:?}, which is not us."); + } + } + Request::Query(query) => { + self.send_event(NetworkEvent::QueryRequestReceived { + query, + channel: MsgResponder::FromPeer(channel), + }) + } + } + } + Message::Response { + request_id, + response, + } => { + trace!("Got response {request_id:?} from peer {peer:?}, res: {response}."); + if let Some(sender) = self.pending_requests.remove(&request_id) { + // The sender will be provided if the caller (Requester) is awaiting for a response + // at the call site. + // Else the Request was just sent to the peer and the Response was + // meant to be handled in another way and is not awaited. + match sender { + Some(sender) => sender + .send(Ok(response)) + .map_err(|_| NetworkError::InternalMsgChannelDropped)?, + None => { + if let Response::Cmd(CmdResponse::Replicate(Ok(()))) = response { + // Nothing to do, response was fine + // This only exists to ensure we dont drop the handle and + // exit early, potentially logging false connection woes + } else { + // responses that are not awaited at the call site must be handled + // separately + self.send_event(NetworkEvent::ResponseReceived { + res: response, + }); + } + } + } + } else { + warn!("Tried to remove a RequestId from pending_requests which was not inserted in the first place. + Use Cmd::SendRequest with sender:None if you want the Response to be fed into the common handle_response function"); + } + } + }, + request_response::Event::OutboundFailure { + request_id, + error, + peer, + } => { + if let Some(sender) = self.pending_requests.remove(&request_id) { + match sender { + Some(sender) => { + sender + .send(Err(error.into())) + .map_err(|_| NetworkError::InternalMsgChannelDropped)?; + } + None => { + warn!("RequestResponse: OutboundFailure for request_id: {request_id:?} and peer: {peer:?}, with error: {error:?}"); + return Err(NetworkError::ReceivedResponseDropped(request_id)); + } + } + } else { + warn!("RequestResponse: OutboundFailure for request_id: {request_id:?} and peer: {peer:?}, with error: {error:?}"); + return Err(NetworkError::ReceivedResponseDropped(request_id)); + } + } + request_response::Event::InboundFailure { + peer, + request_id, + error, + } => { + warn!("RequestResponse: InboundFailure for request_id: {request_id:?} and peer: {peer:?}, with error: {error:?}"); + } + request_response::Event::ResponseSent { peer, request_id } => { + trace!("ResponseSent for request_id: {request_id:?} and peer: {peer:?}"); + } + } + Ok(()) + } + + fn add_keys_to_replication_fetcher( + &mut self, + sender: NetworkAddress, + incoming_keys: Vec<(NetworkAddress, RecordType)>, + ) { + let holder = if let Some(peer_id) = sender.as_peer_id() { + peer_id + } else { + warn!("Replication list sender is not a peer_id {sender:?}"); + return; + }; + + trace!( + "Received replication list from {holder:?} of {} keys", + incoming_keys.len() + ); + + // accept replication requests from the K_VALUE peers away, + // giving us some margin for replication + let closest_k_peers = self.get_closest_k_value_local_peers(); + if !closest_k_peers.contains(&holder) || holder == self.self_peer_id { + trace!("Holder {holder:?} is self or not in replication range."); + return; + } + + // On receive a replication_list from a close_group peer, we undertake two tasks: + // 1, For those keys that we don't have: + // fetch them if close enough to us + // 2, For those keys that we have and supposed to be held by the sender as well: + // start chunk_proof check against a randomly selected chunk type record to the sender + + // For fetching, only handle those non-exist and in close range keys + let keys_to_store = + self.select_non_existent_records_for_replications(&incoming_keys, &closest_k_peers); + + if keys_to_store.is_empty() { + debug!("Empty keys to store after adding to"); + } else { + #[allow(clippy::mutable_key_type)] + let all_keys = self + .swarm + .behaviour_mut() + .kademlia + .store_mut() + .record_addresses_ref(); + let keys_to_fetch = self + .replication_fetcher + .add_keys(holder, keys_to_store, all_keys); + if keys_to_fetch.is_empty() { + trace!("no waiting keys to fetch from the network"); + } else { + self.send_event(NetworkEvent::KeysToFetchForReplication(keys_to_fetch)); + } + } + + // Only trigger chunk_proof check when received a periodical replication request. + if incoming_keys.len() > 1 { + let keys_to_verify = self.select_verification_data_candidates(sender); + + if keys_to_verify.is_empty() { + debug!("No valid candidate to be checked against peer {holder:?}"); + } else { + self.send_event(NetworkEvent::ChunkProofVerification { + peer_id: holder, + keys_to_verify, + }); + } + } + } + + /// Checks suggested records against what we hold, so we only + /// enqueue what we do not have + fn select_non_existent_records_for_replications( + &mut self, + incoming_keys: &[(NetworkAddress, RecordType)], + closest_k_peers: &Vec, + ) -> Vec<(NetworkAddress, RecordType)> { + #[allow(clippy::mutable_key_type)] + let locally_stored_keys = self + .swarm + .behaviour_mut() + .kademlia + .store_mut() + .record_addresses_ref(); + let non_existent_keys: Vec<_> = incoming_keys + .iter() + .filter(|(addr, record_type)| { + let key = addr.to_record_key(); + let local = locally_stored_keys.get(&key); + + // if we have a local value of matching record_type, we don't need to fetch it + if let Some((_, local_record_type)) = local { + let not_same_type = local_record_type != record_type; + if not_same_type { + // Shall only happens for Register + info!("Record {addr:?} has different type: local {local_record_type:?}, incoming {record_type:?}"); + } + not_same_type + } else { + true + } + }) + .collect(); + + non_existent_keys + .into_iter() + .filter_map(|(key, record_type)| { + if Self::is_in_close_range(&self.self_peer_id, key, closest_k_peers) { + Some((key.clone(), record_type.clone())) + } else { + // Reduce the log level as there will always be around 40% records being + // out of the close range, as the sender side is using `CLOSE_GROUP_SIZE + 2` + // to send our replication list to provide addressing margin. + // Given there will normally be 6 nodes sending such list with interval of 5-10s, + // this will accumulate to a lot of logs with the increasing records uploaded. + trace!("not in close range for key {key:?}"); + None + } + }) + .collect() + } + + /// A close target doesn't falls into the close peers range: + /// For example, a node b11111X has an RT: [(1, b1111), (2, b111), (5, b11), (9, b1), (7, b0)] + /// Then for a target bearing b011111 as prefix, all nodes in (7, b0) are its close_group peers. + /// Then the node b11111X. But b11111X's close_group peers [(1, b1111), (2, b111), (5, b11)] + /// are none among target b011111's close range. + /// Hence, the ilog2 calculation based on close_range cannot cover such case. + /// And have to sort all nodes to figure out whether self is among the close_group to the target. + fn is_in_close_range( + our_peer_id: &PeerId, + target: &NetworkAddress, + all_peers: &Vec, + ) -> bool { + if all_peers.len() <= REPLICATION_PEERS_COUNT { + return true; + } + + // Margin of 2 to allow our RT being bit lagging. + match sort_peers_by_address(all_peers, target, REPLICATION_PEERS_COUNT) { + Ok(close_group) => close_group.contains(&our_peer_id), + Err(err) => { + warn!("Could not get sorted peers for {target:?} with error {err:?}"); + true + } + } + } + + /// Check among all chunk type records that we have, select those close to the peer, + /// and randomly pick one as the verification candidate. + #[allow(clippy::mutable_key_type)] + fn select_verification_data_candidates(&mut self, peer: NetworkAddress) -> Vec { + let mut closest_peers = self + .swarm + .behaviour_mut() + .kademlia + .get_closest_local_peers(&self.self_peer_id.into()) + .map(|peer| peer.into_preimage()) + .take(20) + .collect_vec(); + closest_peers.push(self.self_peer_id); + + let target_peer = if let Some(peer_id) = peer.as_peer_id() { + peer_id + } else { + error!("Target {peer:?} is not a valid PeerId"); + return vec![]; + }; + + #[allow(clippy::mutable_key_type)] + let all_keys = self + .swarm + .behaviour_mut() + .kademlia + .store_mut() + .record_addresses_ref(); + + // Targeted chunk type record shall be expected within the close range from our perspective. + let mut verify_candidates: Vec = all_keys + .values() + .filter_map(|(addr, record_type)| { + if RecordType::Chunk == *record_type { + match sort_peers_by_address(&closest_peers, addr, CLOSE_GROUP_SIZE) { + Ok(close_group) => { + if close_group.contains(&&target_peer) { + Some(addr.clone()) + } else { + None + } + } + Err(err) => { + warn!("Could not get sorted peers for {addr:?} with error {err:?}"); + None + } + } + } else { + None + } + }) + .collect(); + + verify_candidates.sort_by_key(|a| peer.distance(a)); + + // To ensure the candidate mush have to be held by the peer, + // we only carry out check when there are already certain amount of chunks uploaded + // AND choose candidate from certain reduced range. + if verify_candidates.len() > 50 { + let index: usize = OsRng.gen_range(0..(verify_candidates.len() / 2)); + vec![verify_candidates[index].clone()] + } else { + vec![] + } + } +} diff --git a/sn_networking/src/event/swarm.rs b/sn_networking/src/event/swarm.rs new file mode 100644 index 0000000000..70b6681271 --- /dev/null +++ b/sn_networking/src/event/swarm.rs @@ -0,0 +1,644 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::{ + cmd::SwarmCmd, event::NodeEvent, multiaddr_is_global, multiaddr_strip_p2p, + target_arch::Instant, NetworkEvent, Result, SwarmDriver, +}; +use itertools::Itertools; +#[cfg(feature = "local-discovery")] +use libp2p::mdns; +use libp2p::{ + kad::K_VALUE, + multiaddr::Protocol, + swarm::{ + dial_opts::{DialOpts, PeerCondition}, + DialError, SwarmEvent, + }, + Multiaddr, PeerId, TransportError, +}; +use sn_protocol::{ + get_port_from_multiaddr, + version::{IDENTIFY_NODE_VERSION_STR, IDENTIFY_PROTOCOL_STR}, +}; +use std::collections::HashSet; +use tokio::time::Duration; + +impl SwarmDriver { + /// Handle `SwarmEvents` + pub(crate) fn handle_swarm_events(&mut self, event: SwarmEvent) -> Result<()> { + let start = Instant::now(); + let event_string; + match event { + SwarmEvent::Behaviour(NodeEvent::MsgReceived(event)) => { + event_string = "msg_received"; + if let Err(e) = self.handle_req_resp_events(event) { + warn!("MsgReceivedError: {e:?}"); + } + } + SwarmEvent::Behaviour(NodeEvent::Kademlia(kad_event)) => { + event_string = "kad_event"; + self.handle_kad_event(kad_event)?; + } + SwarmEvent::Behaviour(NodeEvent::Dcutr(event)) => { + event_string = "dcutr_event"; + info!( + "Dcutr with remote peer: {:?} is: {:?}", + event.remote_peer_id, event.result + ); + } + SwarmEvent::Behaviour(NodeEvent::RelayClient(event)) => { + event_string = "relay_client_event"; + + info!(?event, "relay client event"); + + if let libp2p::relay::client::Event::ReservationReqAccepted { + relay_peer_id, .. + } = *event + { + self.relay_manager + .on_successful_reservation_by_client(&relay_peer_id, &mut self.swarm); + } + } + + SwarmEvent::Behaviour(NodeEvent::RelayServer(event)) => { + event_string = "relay_server_event"; + + info!(?event, "relay server event"); + + match *event { + libp2p::relay::Event::ReservationReqAccepted { + src_peer_id, + renewed: _, + } => { + self.relay_manager + .on_successful_reservation_by_server(src_peer_id); + } + libp2p::relay::Event::ReservationTimedOut { src_peer_id } => { + self.relay_manager.on_reservation_timeout(src_peer_id); + } + _ => {} + } + } + SwarmEvent::Behaviour(NodeEvent::Identify(iden)) => { + event_string = "identify"; + + match *iden { + libp2p::identify::Event::Received { peer_id, info } => { + trace!(%peer_id, ?info, "identify: received info"); + + if info.protocol_version != IDENTIFY_PROTOCOL_STR.to_string() { + warn!(?info.protocol_version, "identify: {peer_id:?} does not have the same protocol. Our IDENTIFY_PROTOCOL_STR: {:?}", IDENTIFY_PROTOCOL_STR.as_str()); + + self.send_event(NetworkEvent::PeerWithUnsupportedProtocol { + our_protocol: IDENTIFY_PROTOCOL_STR.to_string(), + their_protocol: info.protocol_version, + }); + + return Ok(()); + } + + // if client, return. + if info.agent_version != IDENTIFY_NODE_VERSION_STR.to_string() { + return Ok(()); + } + + let has_dialed = self.dialed_peers.contains(&peer_id); + + // If we're not in local mode, only add globally reachable addresses. + // Strip the `/p2p/...` part of the multiaddresses. + // Collect into a HashSet directly to avoid multiple allocations and handle deduplication. + let addrs: HashSet = match self.local { + true => info + .listen_addrs + .into_iter() + .map(|addr| multiaddr_strip_p2p(&addr)) + .collect(), + false => info + .listen_addrs + .into_iter() + .filter(multiaddr_is_global) + .map(|addr| multiaddr_strip_p2p(&addr)) + .collect(), + }; + + self.relay_manager.add_potential_candidates( + &peer_id, + &addrs, + &info.protocols, + ); + + // When received an identify from un-dialed peer, try to dial it + // The dial shall trigger the same identify to be sent again and confirm + // peer is external accessible, hence safe to be added into RT. + if !self.local && !has_dialed { + // Only need to dial back for not fulfilled kbucket + let (kbucket_full, already_present_in_rt, ilog2) = + if let Some(kbucket) = + self.swarm.behaviour_mut().kademlia.kbucket(peer_id) + { + let ilog2 = kbucket.range().0.ilog2(); + let num_peers = kbucket.num_entries(); + let mut is_bucket_full = num_peers >= K_VALUE.into(); + + // check if peer_id is already a part of RT + let already_present_in_rt = kbucket + .iter() + .any(|entry| entry.node.key.preimage() == &peer_id); + + // If the bucket contains any of a bootstrap node, + // consider the bucket is not full and dial back + // so that the bootstrap nodes can be replaced. + if is_bucket_full { + if let Some(peers) = self.bootstrap_peers.get(&ilog2) { + if kbucket.iter().any(|entry| { + peers.contains(entry.node.key.preimage()) + }) { + is_bucket_full = false; + } + } + } + + (is_bucket_full, already_present_in_rt, ilog2) + } else { + return Ok(()); + }; + + if kbucket_full { + trace!("received identify for a full bucket {ilog2:?}, not dialing {peer_id:?} on {addrs:?}"); + return Ok(()); + } else if already_present_in_rt { + trace!("received identify for {peer_id:?} that is already part of the RT. Not dialing {peer_id:?} on {addrs:?}"); + return Ok(()); + } + + info!(%peer_id, ?addrs, "received identify info from undialed peer for not full kbucket {ilog2:?}, dial back to confirm external accessible"); + if let Err(err) = self.swarm.dial( + DialOpts::peer_id(peer_id) + .condition(PeerCondition::NotDialing) + .addresses(addrs.iter().cloned().collect()) + .build(), + ) { + warn!(%peer_id, ?addrs, "dialing error: {err:?}"); + } + + trace!( + "SwarmEvent handled in {:?}: {event_string:?}", + start.elapsed() + ); + return Ok(()); + } + + // If we are not local, we care only for peers that we dialed and thus are reachable. + if self.local || has_dialed { + // To reduce the bad_node check resource usage, + // during the connection establish process, only check cached black_list + // The periodical check, which involves network queries shall filter + // out bad_nodes eventually. + if let Some((_issues, true)) = self.bad_nodes.get(&peer_id) { + info!("Peer {peer_id:?} is considered as bad, blocking it."); + } else { + self.remove_bootstrap_from_full(peer_id); + + trace!(%peer_id, ?addrs, "identify: attempting to add addresses to routing table"); + + // Attempt to add the addresses to the routing table. + for multiaddr in addrs { + let _routing_update = self + .swarm + .behaviour_mut() + .kademlia + .add_address(&peer_id, multiaddr); + } + } + } + trace!( + "SwarmEvent handled in {:?}: {event_string:?}", + start.elapsed() + ); + } + // Log the other Identify events. + libp2p::identify::Event::Sent { .. } => trace!("identify: {iden:?}"), + libp2p::identify::Event::Pushed { .. } => trace!("identify: {iden:?}"), + libp2p::identify::Event::Error { .. } => trace!("identify: {iden:?}"), + } + } + #[cfg(feature = "local-discovery")] + SwarmEvent::Behaviour(NodeEvent::Mdns(mdns_event)) => { + event_string = "mdns"; + match *mdns_event { + mdns::Event::Discovered(list) => { + if self.local { + for (peer_id, addr) in list { + // The multiaddr does not contain the peer ID, so add it. + let addr = addr.with(Protocol::P2p(peer_id)); + + info!(%addr, "mDNS node discovered and dialing"); + + if let Err(err) = self.dial(addr.clone()) { + warn!(%addr, "mDNS node dial error: {err:?}"); + } + } + } + } + mdns::Event::Expired(peer) => { + trace!("mdns peer {peer:?} expired"); + } + } + } + + SwarmEvent::NewListenAddr { + address, + listener_id, + } => { + event_string = "new listen addr"; + + // update our stored port if it is configured to be 0 or None + match self.listen_port { + Some(0) | None => { + if let Some(actual_port) = get_port_from_multiaddr(&address) { + info!("Our listen port is configured as 0 or is not set. Setting it to our actual port: {actual_port}"); + self.listen_port = Some(actual_port); + } + } + _ => {} + }; + + let local_peer_id = *self.swarm.local_peer_id(); + let address = address.with(Protocol::P2p(local_peer_id)); + + // Trigger server mode if we're not a client and we should not add our own address if we're behind + // home network. + if !self.is_client && !self.is_behind_home_network { + if self.local { + // all addresses are effectively external here... + // this is needed for Kad Mode::Server + self.swarm.add_external_address(address.clone()); + } else { + // only add our global addresses + if multiaddr_is_global(&address) { + self.swarm.add_external_address(address.clone()); + } + } + } + + self.send_event(NetworkEvent::NewListenAddr(address.clone())); + + info!("Local node is listening {listener_id:?} on {address:?}"); + } + SwarmEvent::ListenerClosed { + listener_id, + addresses, + reason, + } => { + event_string = "listener closed"; + info!("Listener {listener_id:?} with add {addresses:?} has been closed for {reason:?}"); + self.relay_manager + .on_listener_closed(&listener_id, &mut self.swarm); + } + SwarmEvent::IncomingConnection { + connection_id, + local_addr, + send_back_addr, + } => { + event_string = "incoming"; + trace!("IncomingConnection ({connection_id:?}) with local_addr: {local_addr:?} send_back_addr: {send_back_addr:?}"); + } + SwarmEvent::ConnectionEstablished { + peer_id, + endpoint, + num_established, + connection_id, + concurrent_dial_errors, + established_in, + } => { + event_string = "ConnectionEstablished"; + trace!(%peer_id, num_established, ?concurrent_dial_errors, "ConnectionEstablished ({connection_id:?}) in {established_in:?}: {}", endpoint_str(&endpoint)); + + let _ = self.live_connected_peers.insert( + connection_id, + (peer_id, Instant::now() + Duration::from_secs(60)), + ); + + if endpoint.is_dialer() { + self.dialed_peers.push(peer_id); + } + } + SwarmEvent::ConnectionClosed { + peer_id, + endpoint, + cause, + num_established, + connection_id, + } => { + event_string = "ConnectionClosed"; + trace!(%peer_id, ?connection_id, ?cause, num_established, "ConnectionClosed: {}", endpoint_str(&endpoint)); + let _ = self.live_connected_peers.remove(&connection_id); + } + SwarmEvent::OutgoingConnectionError { + connection_id, + peer_id: None, + error, + } => { + event_string = "OutgoingConnErr"; + warn!("OutgoingConnectionError to on {connection_id:?} - {error:?}"); + } + SwarmEvent::OutgoingConnectionError { + peer_id: Some(failed_peer_id), + error, + connection_id, + } => { + event_string = "OutgoingConnErr"; + warn!("OutgoingConnectionError to {failed_peer_id:?} on {connection_id:?} - {error:?}"); + + // we need to decide if this was a critical error and the peer should be removed from the routing table + let should_clean_peer = match error { + DialError::Transport(errors) => { + // as it's an outgoing error, if it's transport based we can assume it is _our_ fault + // + // (eg, could not get a port for a tcp connection) + // so we default to it not being a real issue + // unless there are _specific_ errors (connection refused eg) + error!("Dial errors len : {:?}", errors.len()); + let mut there_is_a_serious_issue = false; + for (_addr, err) in errors { + error!("OutgoingTransport error : {err:?}"); + + match err { + TransportError::MultiaddrNotSupported(addr) => { + warn!("Multiaddr not supported : {addr:?}"); + // if we can't dial a peer on a given address, we should remove it from the routing table + there_is_a_serious_issue = true + } + TransportError::Other(err) => { + let problematic_errors = [ + "ConnectionRefused", + "HostUnreachable", + "HandshakeTimedOut", + ]; + + let is_bootstrap_peer = self + .bootstrap_peers + .iter() + .any(|(_ilog2, peers)| peers.contains(&failed_peer_id)); + + if is_bootstrap_peer + && self.connected_peers < self.bootstrap_peers.len() + { + warn!("OutgoingConnectionError: On bootstrap peer {failed_peer_id:?}, while still in bootstrap mode, ignoring"); + there_is_a_serious_issue = false; + } else { + // It is really difficult to match this error, due to being eg: + // Custom { kind: Other, error: Left(Left(Os { code: 61, kind: ConnectionRefused, message: "Connection refused" })) } + // if we can match that, let's. But meanwhile we'll check the message + let error_msg = format!("{err:?}"); + if problematic_errors + .iter() + .any(|err| error_msg.contains(err)) + { + warn!("Problematic error encountered: {error_msg}"); + there_is_a_serious_issue = true; + } + } + } + } + } + there_is_a_serious_issue + } + DialError::NoAddresses => { + // We provided no address, and while we can't really blame the peer + // we also can't connect, so we opt to cleanup... + warn!("OutgoingConnectionError: No address provided"); + true + } + DialError::Aborted => { + // not their fault + warn!("OutgoingConnectionError: Aborted"); + false + } + DialError::DialPeerConditionFalse(_) => { + // we could not dial due to an internal condition, so not their issue + warn!("OutgoingConnectionError: DialPeerConditionFalse"); + false + } + DialError::LocalPeerId { endpoint, .. } => { + // This is actually _us_ So we should remove this from the RT + error!( + "OutgoingConnectionError: LocalPeerId: {}", + endpoint_str(&endpoint) + ); + true + } + DialError::WrongPeerId { obtained, endpoint } => { + // The peer id we attempted to dial was not the one we expected + // cleanup + error!("OutgoingConnectionError: WrongPeerId: obtained: {obtained:?}, endpoint: {endpoint:?}"); + true + } + DialError::Denied { cause } => { + // The peer denied our connection + // cleanup + error!("OutgoingConnectionError: Denied: {cause:?}"); + true + } + }; + + if should_clean_peer { + warn!("Tracking issue of {failed_peer_id:?}. Clearing it out for now"); + + if let Some(dead_peer) = self + .swarm + .behaviour_mut() + .kademlia + .remove_peer(&failed_peer_id) + { + self.connected_peers = self.connected_peers.saturating_sub(1); + + self.handle_cmd(SwarmCmd::RecordNodeIssue { + peer_id: failed_peer_id, + issue: crate::NodeIssue::ConnectionIssue, + })?; + + self.send_event(NetworkEvent::PeerRemoved( + *dead_peer.node.key.preimage(), + self.connected_peers, + )); + + self.log_kbuckets(&failed_peer_id); + let _ = self.check_for_change_in_our_close_group(); + } + } + } + SwarmEvent::IncomingConnectionError { + connection_id, + local_addr, + send_back_addr, + error, + } => { + event_string = "Incoming ConnErr"; + error!("IncomingConnectionError from local_addr:?{local_addr:?}, send_back_addr {send_back_addr:?} on {connection_id:?} with error {error:?}"); + } + SwarmEvent::Dialing { + peer_id, + connection_id, + } => { + event_string = "Dialing"; + trace!("Dialing {peer_id:?} on {connection_id:?}"); + } + SwarmEvent::NewExternalAddrCandidate { address } => { + event_string = "NewExternalAddrCandidate"; + + if !self.swarm.external_addresses().any(|addr| addr == &address) + && !self.is_client + // If we are behind a home network, then our IP is returned here. We should be only having + // relay server as our external address + // todo: can our relay address be reported here? If so, maybe we should add them. + && !self.is_behind_home_network + { + debug!(%address, "external address: new candidate"); + + // Identify will let us know when we have a candidate. (Peers will tell us what address they see us as.) + // We manually confirm this to be our externally reachable address, though in theory it's possible we + // are not actually reachable. This event returns addresses with ports that were not set by the user, + // so we must not add those ports as they will not be forwarded. + // Setting this will also switch kad to server mode if it's not already in it. + if let Some(our_port) = self.listen_port { + if let Some(port) = get_port_from_multiaddr(&address) { + if port == our_port { + info!(%address, "external address: new candidate has the same configured port, adding it."); + self.swarm.add_external_address(address); + } else { + info!(%address, %our_port, "external address: new candidate has a different port, not adding it."); + } + } + } else { + trace!("external address: listen port not set. This has to be set if you're running a node"); + } + } + let all_external_addresses = self.swarm.external_addresses().collect_vec(); + let all_listeners = self.swarm.listeners().collect_vec(); + debug!("All our listeners: {all_listeners:?}"); + debug!("All our external addresses: {all_external_addresses:?}"); + } + SwarmEvent::ExternalAddrConfirmed { address } => { + event_string = "ExternalAddrConfirmed"; + info!(%address, "external address: confirmed"); + } + SwarmEvent::ExternalAddrExpired { address } => { + event_string = "ExternalAddrExpired"; + info!(%address, "external address: expired"); + } + other => { + event_string = "Other"; + + trace!("SwarmEvent has been ignored: {other:?}") + } + } + self.remove_outdated_connections(); + + self.log_handling(event_string.to_string(), start.elapsed()); + + trace!( + "SwarmEvent handled in {:?}: {event_string:?}", + start.elapsed() + ); + Ok(()) + } + + // if target bucket is full, remove a bootstrap node if presents. + fn remove_bootstrap_from_full(&mut self, peer_id: PeerId) { + let mut shall_removed = None; + + if let Some(kbucket) = self.swarm.behaviour_mut().kademlia.kbucket(peer_id) { + if kbucket.num_entries() >= K_VALUE.into() { + if let Some(peers) = self.bootstrap_peers.get(&kbucket.range().0.ilog2()) { + for peer_entry in kbucket.iter() { + if peers.contains(peer_entry.node.key.preimage()) { + shall_removed = Some(*peer_entry.node.key.preimage()); + break; + } + } + } + } + } + if let Some(to_be_removed_bootstrap) = shall_removed { + trace!("Bootstrap node {to_be_removed_bootstrap:?} to be replaced by peer {peer_id:?}"); + let _entry = self + .swarm + .behaviour_mut() + .kademlia + .remove_peer(&to_be_removed_bootstrap); + } + } + + // Remove outdated connection to a peer if it is not in the RT. + fn remove_outdated_connections(&mut self) { + let mut shall_removed = vec![]; + + let timed_out_connections = + self.live_connected_peers + .iter() + .filter_map(|(connection_id, (peer_id, timeout))| { + if Instant::now() > *timeout { + Some((connection_id, peer_id)) + } else { + None + } + }); + + for (connection_id, peer_id) in timed_out_connections { + // Skip if the peer is present in our RT + if let Some(kbucket) = self.swarm.behaviour_mut().kademlia.kbucket(*peer_id) { + if kbucket + .iter() + .any(|peer_entry| *peer_id == *peer_entry.node.key.preimage()) + { + continue; + } + } + + // skip if the peer is a relay server that we're connected to + if self.relay_manager.keep_alive_peer(peer_id) { + continue; + } + + shall_removed.push((*connection_id, *peer_id)); + } + + if !shall_removed.is_empty() { + trace!( + "Current libp2p peers pool stats is {:?}", + self.swarm.network_info() + ); + trace!( + "Removing {} outdated live connections, still have {} left.", + shall_removed.len(), + self.live_connected_peers.len() + ); + trace!(?self.relay_manager); + + for (connection_id, peer_id) in shall_removed { + let _ = self.live_connected_peers.remove(&connection_id); + let result = self.swarm.close_connection(connection_id); + trace!("Removed outdated connection {connection_id:?} to {peer_id:?} with result: {result:?}"); + } + } + } +} + +/// Helper function to print formatted connection role info. +fn endpoint_str(endpoint: &libp2p::core::ConnectedPoint) -> String { + match endpoint { + libp2p::core::ConnectedPoint::Dialer { address, .. } => { + format!("outgoing ({address})") + } + libp2p::core::ConnectedPoint::Listener { send_back_addr, .. } => { + format!("incoming ({send_back_addr})") + } + } +} diff --git a/sn_networking/src/get_record_handler.rs b/sn_networking/src/get_record_handler.rs deleted file mode 100644 index 574ad32293..0000000000 --- a/sn_networking/src/get_record_handler.rs +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use crate::{ - get_quorum_value, GetRecordCfg, GetRecordError, NetworkError, Result, SwarmDriver, - CLOSE_GROUP_SIZE, -}; -use libp2p::{ - kad::{self, PeerRecord, ProgressStep, QueryId, QueryStats, Record}, - PeerId, -}; -use sn_protocol::PrettyPrintRecordKey; -use std::collections::{hash_map::Entry, HashMap, HashSet}; -use tokio::sync::oneshot; -use xor_name::XorName; - -/// Using XorName to differentiate different record content under the same key. -type GetRecordResultMap = HashMap)>; -pub(crate) type PendingGetRecord = HashMap< - QueryId, - ( - oneshot::Sender>, - GetRecordResultMap, - GetRecordCfg, - ), ->; - -// For `get_record` returning behaviour: -// 1, targeting a non-existing entry -// there will only be one event of `kad::Event::OutboundQueryProgressed` -// with `ProgressStep::last` to be `true` -// `QueryStats::requests` to be 20 (K-Value) -// `QueryStats::success` to be over majority of the requests -// `err::NotFound::closest_peers` contains a list of CLOSE_GROUP_SIZE peers -// 2, targeting an existing entry -// there will a sequence of (at least CLOSE_GROUP_SIZE) events of -// `kad::Event::OutboundQueryProgressed` to be received -// with `QueryStats::end` always being `None` -// `ProgressStep::last` all to be `false` -// `ProgressStep::count` to be increased with step of 1 -// capped and stopped at CLOSE_GROUP_SIZE, may have duplicated counts -// `PeerRecord::peer` could be None to indicate from self -// in which case it always use a duplicated `ProgressStep::count` -// the sequence will be completed with `FinishedWithNoAdditionalRecord` -// where: `cache_candidates`: being the peers supposed to hold the record but not -// `ProgressStep::count`: to be `number of received copies plus one` -// `ProgressStep::last` to be `true` -impl SwarmDriver { - // Accumulates the GetRecord query results - // If we get enough responses (quorum) for a record with the same content hash: - // - we return the Record after comparing with the target record. This might return RecordDoesNotMatch if the - // check fails. - // - if multiple content hashes are found, we return a SplitRecord Error - // And then we stop the kad query as we are done here. - pub(crate) fn accumulate_get_record_found( - &mut self, - query_id: QueryId, - peer_record: PeerRecord, - _stats: QueryStats, - step: ProgressStep, - ) -> Result<()> { - let peer_id = if let Some(peer_id) = peer_record.peer { - peer_id - } else { - self.self_peer_id - }; - let pretty_key = PrettyPrintRecordKey::from(&peer_record.record.key).into_owned(); - - if let Entry::Occupied(mut entry) = self.pending_get_record.entry(query_id) { - let (_sender, result_map, cfg) = entry.get_mut(); - - if !cfg.expected_holders.is_empty() { - if cfg.expected_holders.remove(&peer_id) { - debug!("For record {pretty_key:?} task {query_id:?}, received a copy from an expected holder {peer_id:?}"); - } else { - debug!("For record {pretty_key:?} task {query_id:?}, received a copy from an unexpected holder {peer_id:?}"); - } - } - - // Insert the record and the peer into the result_map. - let record_content_hash = XorName::from_content(&peer_record.record.value); - let responded_peers = - if let Entry::Occupied(mut entry) = result_map.entry(record_content_hash) { - let (_, peer_list) = entry.get_mut(); - let _ = peer_list.insert(peer_id); - peer_list.len() - } else { - let mut peer_list = HashSet::new(); - let _ = peer_list.insert(peer_id); - result_map.insert(record_content_hash, (peer_record.record.clone(), peer_list)); - 1 - }; - - let expected_answers = get_quorum_value(&cfg.get_quorum); - - trace!("Expecting {expected_answers:?} answers for record {pretty_key:?} task {query_id:?}, received {responded_peers} so far"); - - if responded_peers >= expected_answers { - if !cfg.expected_holders.is_empty() { - debug!("For record {pretty_key:?} task {query_id:?}, fetch completed with non-responded expected holders {:?}", cfg.expected_holders); - } - let cfg = cfg.clone(); - - // Remove the query task and consume the variables. - let (sender, result_map, _) = entry.remove(); - - if result_map.len() == 1 { - Self::send_record_after_checking_target(sender, peer_record.record, &cfg)?; - } else { - debug!("For record {pretty_key:?} task {query_id:?}, fetch completed with split record"); - sender - .send(Err(GetRecordError::SplitRecord { result_map })) - .map_err(|_| NetworkError::InternalMsgChannelDropped)?; - } - - // Stop the query; possibly stops more nodes from being queried. - if let Some(mut query) = self.swarm.behaviour_mut().kademlia.query_mut(&query_id) { - query.finish(); - } - } else if usize::from(step.count) >= CLOSE_GROUP_SIZE { - debug!("For record {pretty_key:?} task {query_id:?}, got {:?} with {} versions so far.", - step.count, result_map.len()); - } - } else { - // return error if the entry cannot be found - return Err(NetworkError::ReceivedKademliaEventDropped { - query_id, - event: format!("Accumulate Get Record of {pretty_key:?}"), - }); - } - Ok(()) - } - - // Handles the possible cases when a GetRecord Query completes. - // The accumulate_get_record_found returns the record if the quorum is satisfied, but, if we have reached this point - // then we did not get enough records or we got split records (which prevented the quorum to pass). - // Returns the following errors: - // RecordNotFound if the result_map is empty. - // NotEnoughCopies if there is only a single content hash version. - // SplitRecord if there are multiple content hash versions. - pub(crate) fn handle_get_record_finished( - &mut self, - query_id: QueryId, - step: ProgressStep, - ) -> Result<()> { - // return error if the entry cannot be found - if let Some((sender, result_map, cfg)) = self.pending_get_record.remove(&query_id) { - let num_of_versions = result_map.len(); - let (result, log_string) = if let Some((record, from_peers)) = - result_map.values().next() - { - let result = if num_of_versions == 1 { - Err(GetRecordError::NotEnoughCopies { - record: record.clone(), - expected: get_quorum_value(&cfg.get_quorum), - got: from_peers.len(), - }) - } else { - Err(GetRecordError::SplitRecord { - result_map: result_map.clone(), - }) - }; - - ( - result, - format!("Getting record {:?} completed with only {:?} copies received, and {num_of_versions} versions.", - PrettyPrintRecordKey::from(&record.key), usize::from(step.count) - 1) - ) - } else { - ( - Err(GetRecordError::RecordNotFound), - format!("Getting record task {query_id:?} completed with step count {:?}, but no copy found.", step.count), - ) - }; - - if cfg.expected_holders.is_empty() { - debug!("{log_string}"); - } else { - debug!( - "{log_string}, and {:?} expected holders not responded", - cfg.expected_holders - ); - } - - sender - .send(result) - .map_err(|_| NetworkError::InternalMsgChannelDropped)?; - } else { - // We manually perform `query.finish()` if we return early from accumulate fn. - // Thus we will still get FinishedWithNoAdditionalRecord. - trace!("Can't locate query task {query_id:?} during GetRecord finished. We might have already returned the result to the sender."); - } - Ok(()) - } - - /// Handles the possible cases when a kad GetRecord returns an error. - /// If we get NotFound/QuorumFailed, we return a RecordNotFound error. Kad currently does not enforce any quorum. - /// If we get a Timeout: - /// - return a QueryTimeout if we get a split record (?) if we have multiple content hashes. - /// - if the quorum is satisfied, we return the record after comparing it with the target record. This might return - /// RecordDoesNotMatch if the check fails. - /// - else we return q QueryTimeout error. - pub(crate) fn handle_get_record_error( - &mut self, - query_id: QueryId, - get_record_err: kad::GetRecordError, - _stats: QueryStats, - _step: ProgressStep, - ) -> Result<()> { - match &get_record_err { - kad::GetRecordError::NotFound { .. } | kad::GetRecordError::QuorumFailed { .. } => { - // return error if the entry cannot be found - let (sender, _, cfg) = - self.pending_get_record.remove(&query_id).ok_or_else(|| { - trace!("Can't locate query task {query_id:?}, it has likely been completed already."); - NetworkError::ReceivedKademliaEventDropped { - query_id, - event: "GetRecordError NotFound or QuorumFailed".to_string(), - } - })?; - - if cfg.expected_holders.is_empty() { - info!("Get record task {query_id:?} failed with error {get_record_err:?}"); - } else { - debug!("Get record task {query_id:?} failed with {:?} expected holders not responded, error {get_record_err:?}", cfg.expected_holders); - } - sender - .send(Err(GetRecordError::RecordNotFound)) - .map_err(|_| NetworkError::InternalMsgChannelDropped)?; - } - kad::GetRecordError::Timeout { key } => { - // return error if the entry cannot be found - let pretty_key = PrettyPrintRecordKey::from(key); - let (sender, result_map, cfg) = - self.pending_get_record.remove(&query_id).ok_or_else(|| { - trace!( - "Can't locate query task {query_id:?} for {pretty_key:?}, it has likely been completed already." - ); - NetworkError::ReceivedKademliaEventDropped { - query_id, - event: format!("GetRecordError Timeout {pretty_key:?}"), - } - })?; - - let required_response_count = get_quorum_value(&cfg.get_quorum); - - // if we've a split over the result xorname, then we don't attempt to resolve this here. - // Retry and resolve through normal flows without a timeout. - // todo: is the above still the case? Why don't we return a split record error. - if result_map.len() > 1 { - warn!( - "Get record task {query_id:?} for {pretty_key:?} timed out with split result map" - ); - sender - .send(Err(GetRecordError::QueryTimeout)) - .map_err(|_| NetworkError::InternalMsgChannelDropped)?; - - return Ok(()); - } - - // if we have enough responses here, we can return the record - if let Some((record, peers)) = result_map.values().next() { - if peers.len() >= required_response_count { - Self::send_record_after_checking_target(sender, record.clone(), &cfg)?; - return Ok(()); - } - } - - warn!("Get record task {query_id:?} for {pretty_key:?} returned insufficient responses. {:?} did not return record", cfg.expected_holders); - // Otherwise report the timeout - sender - .send(Err(GetRecordError::QueryTimeout)) - .map_err(|_| NetworkError::InternalMsgChannelDropped)?; - } - } - - Ok(()) - } - - fn send_record_after_checking_target( - sender: oneshot::Sender>, - record: Record, - cfg: &GetRecordCfg, - ) -> Result<()> { - if cfg.target_record.is_none() || cfg.does_target_match(&record) { - sender - .send(Ok(record)) - .map_err(|_| NetworkError::InternalMsgChannelDropped) - } else { - sender - .send(Err(GetRecordError::RecordDoesNotMatch(record))) - .map_err(|_| NetworkError::InternalMsgChannelDropped) - } - } -} diff --git a/sn_networking/src/lib.rs b/sn_networking/src/lib.rs index afbf50930d..75d51ab1e2 100644 --- a/sn_networking/src/lib.rs +++ b/sn_networking/src/lib.rs @@ -15,7 +15,6 @@ mod cmd; mod driver; mod error; mod event; -mod get_record_handler; mod log_markers; #[cfg(feature = "open-metrics")] mod metrics; From 8725365e48de4e59c4a688c7e50e43566abf909f Mon Sep 17 00:00:00 2001 From: qima Date: Thu, 25 Apr 2024 21:36:45 +0800 Subject: [PATCH 170/205] feat(node): make spend and cash_note reason field configurable --- .github/workflows/memcheck.yml | 6 +-- .../src/bin/subcommands/wallet/wo_wallet.rs | 9 ++-- sn_client/src/api.rs | 16 +++++- sn_client/src/audit/spend_dag.rs | 6 ++- sn_client/src/audit/tests/setup.rs | 1 + sn_client/src/uploader/tests/setup.rs | 7 ++- sn_client/src/wallet.rs | 13 +++-- sn_networking/src/driver.rs | 11 +++- sn_networking/src/lib.rs | 17 ++++-- sn_networking/src/transfers.rs | 18 ++++--- sn_node/src/bin/safenode/main.rs | 5 ++ sn_node/src/node.rs | 6 ++- sn_node/src/quote.rs | 10 +++- sn_node/tests/double_spend.rs | 16 +++++- sn_node_manager/src/bin/cli/main.rs | 11 ++++ sn_node_manager/src/cmd/local.rs | 4 ++ sn_node_manager/src/local.rs | 41 +++++++++++++-- sn_transfers/benches/reissue.rs | 3 ++ sn_transfers/src/cashnotes.rs | 6 ++- sn_transfers/src/cashnotes/builder.rs | 28 +++++----- sn_transfers/src/cashnotes/cashnote.rs | 21 ++++++-- sn_transfers/src/genesis.rs | 3 +- sn_transfers/src/lib.rs | 5 +- sn_transfers/src/transfers.rs | 4 +- .../src/transfers/offline_transfer.rs | 36 ++++++++----- sn_transfers/src/transfers/transfer.rs | 17 +++++- sn_transfers/src/wallet/data_payments.rs | 8 +++ sn_transfers/src/wallet/hot_wallet.rs | 52 ++++++++++++++----- sn_transfers/src/wallet/watch_only.rs | 8 +-- 29 files changed, 300 insertions(+), 88 deletions(-) diff --git a/.github/workflows/memcheck.yml b/.github/workflows/memcheck.yml index 3275501204..d7e089b4da 100644 --- a/.github/workflows/memcheck.yml +++ b/.github/workflows/memcheck.yml @@ -46,7 +46,7 @@ jobs: run: | mkdir -p $BOOTSTRAP_NODE_DATA_PATH ./target/release/safenode --first \ - --root-dir $BOOTSTRAP_NODE_DATA_PATH --log-output-dest $BOOTSTRAP_NODE_DATA_PATH --local & + --root-dir $BOOTSTRAP_NODE_DATA_PATH --log-output-dest $BOOTSTRAP_NODE_DATA_PATH --local --owner=maidsafe_test & sleep 10 env: SN_LOG: "all" @@ -65,7 +65,7 @@ jobs: run: | mkdir -p $RESTART_TEST_NODE_DATA_PATH ./target/release/safenode \ - --root-dir $RESTART_TEST_NODE_DATA_PATH --log-output-dest $RESTART_TEST_NODE_DATA_PATH --local & + --root-dir $RESTART_TEST_NODE_DATA_PATH --log-output-dest $RESTART_TEST_NODE_DATA_PATH --local --owner=maidsafe_test & sleep 10 env: SN_LOG: "all" @@ -165,7 +165,7 @@ jobs: - name: Start the restart node again run: | ./target/release/safenode \ - --root-dir $RESTART_TEST_NODE_DATA_PATH --log-output-dest $RESTART_TEST_NODE_DATA_PATH --local & + --root-dir $RESTART_TEST_NODE_DATA_PATH --log-output-dest $RESTART_TEST_NODE_DATA_PATH --local --owner=maidsafe_test & sleep 10 env: SN_LOG: "all" diff --git a/sn_cli/src/bin/subcommands/wallet/wo_wallet.rs b/sn_cli/src/bin/subcommands/wallet/wo_wallet.rs index 73396b9949..99e694d7b2 100644 --- a/sn_cli/src/bin/subcommands/wallet/wo_wallet.rs +++ b/sn_cli/src/bin/subcommands/wallet/wo_wallet.rs @@ -16,7 +16,7 @@ use color_eyre::{ }; use dialoguer::Confirm; use sn_client::transfers::{ - DerivationIndex, MainPubkey, NanoTokens, OfflineTransfer, SignedSpend, UniquePubkey, + CashNoteOutputDetails, MainPubkey, NanoTokens, OfflineTransfer, SignedSpend, UniquePubkey, WatchOnlyWallet, }; use sn_client::Client; @@ -232,7 +232,10 @@ fn build_unsigned_transaction(from: &str, amount: &str, to: &str, root_dir: &Pat } }; - let unsigned_transfer = wallet.build_unsigned_transaction(vec![(amount, to)], None)?; + let unsigned_transfer = wallet.build_unsigned_transaction( + vec![("CASH_NOTE_REASON_FOR_TRANSFER".to_string(), amount, to)], + None, + )?; println!( "The unsigned transaction has been successfully created:\n\n{}\n", @@ -251,7 +254,7 @@ async fn broadcast_signed_spends( ) -> Result<()> { let (signed_spends, output_details, change_id): ( BTreeSet, - BTreeMap, + BTreeMap, UniquePubkey, ) = rmp_serde::from_slice(&hex::decode(signed_tx)?)?; diff --git a/sn_client/src/api.rs b/sn_client/src/api.rs index 11a5cf15fb..44afb567df 100644 --- a/sn_client/src/api.rs +++ b/sn_client/src/api.rs @@ -105,11 +105,23 @@ impl Client { let root_dir = std::env::temp_dir(); trace!("Starting Kad swarm in client mode..{root_dir:?}."); + // TODO: shall client bearing owner's discord user name, to be reflected in the cash_notes? + #[cfg(not(feature = "open-metrics"))] - let network_builder = NetworkBuilder::new(Keypair::generate_ed25519(), local, root_dir); + let network_builder = NetworkBuilder::new( + Keypair::generate_ed25519(), + local, + root_dir, + "maidsafe_client".to_string(), + ); #[cfg(feature = "open-metrics")] - let mut network_builder = NetworkBuilder::new(Keypair::generate_ed25519(), local, root_dir); + let mut network_builder = NetworkBuilder::new( + Keypair::generate_ed25519(), + local, + root_dir, + "maidsafe_client".to_string(), + ); #[cfg(feature = "open-metrics")] network_builder.metrics_registry(Registry::default()); diff --git a/sn_client/src/audit/spend_dag.rs b/sn_client/src/audit/spend_dag.rs index b92fc44ab4..0b5053273f 100644 --- a/sn_client/src/audit/spend_dag.rs +++ b/sn_client/src/audit/spend_dag.rs @@ -347,7 +347,11 @@ impl SpendDag { for s in spends { for derivation_idx in s.spend.network_royalties.iter() { let spend_addr = SpendAddress::from_unique_pubkey(&s.spend.unique_pubkey); - royalties.push(CashNoteRedemption::new(*derivation_idx, spend_addr)); + royalties.push(CashNoteRedemption::new( + *derivation_idx, + spend_addr, + "CASH_NOTE_REASON_FOR_NETWORK_ROYALTIES".to_string(), + )); } } Ok(royalties) diff --git a/sn_client/src/audit/tests/setup.rs b/sn_client/src/audit/tests/setup.rs index cdc7caff80..468ac021aa 100644 --- a/sn_client/src/audit/tests/setup.rs +++ b/sn_client/src/audit/tests/setup.rs @@ -117,6 +117,7 @@ impl MockNetwork { .map_err(|e| eyre!("could not get cashnotes for transfer: {e}"))?; let recipient = vec![( NanoTokens::from(amount), + Default::default(), to_wallet.sk.main_pubkey(), DerivationIndex::random(&mut rng), )]; diff --git a/sn_client/src/uploader/tests/setup.rs b/sn_client/src/uploader/tests/setup.rs index 15dc536685..3e8d678d7c 100644 --- a/sn_client/src/uploader/tests/setup.rs +++ b/sn_client/src/uploader/tests/setup.rs @@ -425,7 +425,12 @@ pub fn start_uploading_with_steps( // Build a very simple client struct for testing. This does not connect to any network. // The UploaderInterface eliminates the need for direct networking in tests. pub fn build_unconnected_client(root_dir: PathBuf) -> Result { - let network_builder = NetworkBuilder::new(Keypair::generate_ed25519(), true, root_dir); + let network_builder = NetworkBuilder::new( + Keypair::generate_ed25519(), + true, + root_dir, + "maidsafe_client".to_string(), + ); let (network, ..) = network_builder.build_client()?; let client = Client { network: network.clone(), diff --git a/sn_client/src/wallet.rs b/sn_client/src/wallet.rs index eb9c9f7108..1f60f09d1e 100644 --- a/sn_client/src/wallet.rs +++ b/sn_client/src/wallet.rs @@ -16,7 +16,7 @@ use sn_networking::target_arch::Instant; use sn_networking::{GetRecordError, PayeeQuote}; use sn_protocol::NetworkAddress; use sn_transfers::{ - CashNote, DerivationIndex, HotWallet, MainPubkey, NanoTokens, Payment, PaymentQuote, + CashNote, CashNoteOutputDetails, HotWallet, MainPubkey, NanoTokens, Payment, PaymentQuote, SignedSpend, SpendAddress, Transaction, Transfer, UniquePubkey, WalletError, WalletResult, }; use std::{ @@ -306,7 +306,10 @@ impl WalletClient { to: MainPubkey, verify_store: bool, ) -> WalletResult { - let created_cash_notes = self.wallet.local_send(vec![(amount, to)], None)?; + let created_cash_notes = self.wallet.local_send( + vec![("CASH_NOTE_REASON_FOR_TRANSFER".to_string(), amount, to)], + None, + )?; // send to network if let Err(error) = self @@ -346,7 +349,7 @@ impl WalletClient { signed_spends: BTreeSet, tx: Transaction, change_id: UniquePubkey, - output_details: BTreeMap, + output_details: BTreeMap, verify_store: bool, ) -> WalletResult { let created_cash_notes = @@ -1086,7 +1089,7 @@ pub async fn send( /// * signed_spends - [BTreeSet]<[SignedSpend]>, /// * transaction - [Transaction], /// * change_id - [UniquePubkey], -/// * output_details - [BTreeMap]<[UniquePubkey], ([MainPubkey], [DerivationIndex])>, +/// * output_details - [BTreeMap]<[UniquePubkey], CashNoteOutputDetails>, /// * verify_store - Boolean. Set to true for mandatory verification via a GET request through a Spend on the network. /// /// # Return value @@ -1128,7 +1131,7 @@ pub async fn broadcast_signed_spends( signed_spends: BTreeSet, tx: Transaction, change_id: UniquePubkey, - output_details: BTreeMap, + output_details: BTreeMap, verify_store: bool, ) -> WalletResult { let mut wallet_client = WalletClient::new(client.clone(), from); diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index c104ec14cb..1d1af4b7fc 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -206,10 +206,11 @@ pub struct NetworkBuilder { metrics_registry: Option, #[cfg(feature = "open-metrics")] metrics_server_port: u16, + owner: String, } impl NetworkBuilder { - pub fn new(keypair: Keypair, local: bool, root_dir: PathBuf) -> Self { + pub fn new(keypair: Keypair, local: bool, root_dir: PathBuf, owner: String) -> Self { Self { is_behind_home_network: false, keypair, @@ -223,6 +224,7 @@ impl NetworkBuilder { metrics_registry: None, #[cfg(feature = "open-metrics")] metrics_server_port: 0, + owner, } } @@ -315,6 +317,7 @@ impl NetworkBuilder { }; let listen_addr = self.listen_addr; + let owner = self.owner.clone(); let (network, events_receiver, mut swarm_driver) = self.build( kad_cfg, @@ -322,6 +325,7 @@ impl NetworkBuilder { false, ProtocolSupport::Full, IDENTIFY_NODE_VERSION_STR.to_string(), + owner, )?; // Listen on the provided address @@ -367,12 +371,15 @@ impl NetworkBuilder { .ok_or_else(|| NetworkError::InvalidCloseGroupSize)?, ); + let owner = self.owner.clone(); + let (network, net_event_recv, driver) = self.build( kad_cfg, None, true, ProtocolSupport::Outbound, IDENTIFY_CLIENT_VERSION_STR.to_string(), + owner, )?; Ok((network, net_event_recv, driver)) @@ -386,6 +393,7 @@ impl NetworkBuilder { is_client: bool, req_res_protocol: ProtocolSupport, identify_version: String, + owner: String, ) -> Result<(Network, mpsc::Receiver, SwarmDriver)> { let peer_id = PeerId::from(self.keypair.public()); // vdash metric (if modified please notify at https://github.com/happybeing/vdash/issues): @@ -579,6 +587,7 @@ impl NetworkBuilder { peer_id: Arc::new(peer_id), root_dir_path: Arc::new(self.root_dir), keypair: Arc::new(self.keypair), + owner, }, network_event_receiver, swarm_driver, diff --git a/sn_networking/src/lib.rs b/sn_networking/src/lib.rs index 75d51ab1e2..23943e91c2 100644 --- a/sn_networking/src/lib.rs +++ b/sn_networking/src/lib.rs @@ -158,9 +158,16 @@ pub struct Network { pub peer_id: Arc, pub root_dir_path: Arc, keypair: Arc, + /// node owner's discord username, in readable format + owner: String, } impl Network { + /// Returns of the discord user_name of the node's owner + pub fn owner(&self) -> String { + self.owner.clone() + } + /// Signs the given data with the node's keypair. pub fn sign(&self, msg: &[u8]) -> Result> { self.keypair.sign(msg).map_err(NetworkError::from) @@ -1046,9 +1053,13 @@ mod tests { #[test] fn test_network_sign_verify() -> eyre::Result<()> { - let (network, _, _) = - NetworkBuilder::new(Keypair::generate_ed25519(), false, std::env::temp_dir()) - .build_client()?; + let (network, _, _) = NetworkBuilder::new( + Keypair::generate_ed25519(), + false, + std::env::temp_dir(), + "maidsafe_test".to_string(), + ) + .build_client()?; let msg = b"test message"; let sig = network.sign(msg)?; assert!(network.verify(msg, &sig)); diff --git a/sn_networking/src/transfers.rs b/sn_networking/src/transfers.rs index 15e739a595..5886281892 100644 --- a/sn_networking/src/transfers.rs +++ b/sn_networking/src/transfers.rs @@ -143,16 +143,17 @@ impl Network { parent_spends.iter().map(|s| s.spent_tx()).collect(); // get our outputs from Tx - let our_output_unique_pubkeys: Vec<(UniquePubkey, DerivationIndex)> = cashnote_redemptions - .iter() - .map(|u| { - let unique_pubkey = main_pubkey.new_unique_pubkey(&u.derivation_index); - (unique_pubkey, u.derivation_index) - }) - .collect(); + let our_output_unique_pubkeys: Vec<(String, UniquePubkey, DerivationIndex)> = + cashnote_redemptions + .iter() + .map(|u| { + let unique_pubkey = main_pubkey.new_unique_pubkey(&u.derivation_index); + (u.reason.clone(), unique_pubkey, u.derivation_index) + }) + .collect(); let mut our_output_cash_notes = Vec::new(); - for (id, derivation_index) in our_output_unique_pubkeys.into_iter() { + for (reason, id, derivation_index) in our_output_unique_pubkeys.into_iter() { let src_tx = parent_txs .iter() .find(|tx| tx.outputs.iter().any(|o| o.unique_pubkey() == &id)) @@ -169,6 +170,7 @@ impl Network { unique_pubkey: id, parent_tx: src_tx, parent_spends: signed_spends, + reason: reason.clone(), main_pubkey, derivation_index, }; diff --git a/sn_node/src/bin/safenode/main.rs b/sn_node/src/bin/safenode/main.rs index 9ff693dc46..e4d23116e2 100644 --- a/sn_node/src/bin/safenode/main.rs +++ b/sn_node/src/bin/safenode/main.rs @@ -150,6 +150,10 @@ struct Opt { #[clap(long)] local: bool, + /// Specify the owner(readable discord user name). + #[clap(long)] + owner: String, + #[cfg(feature = "open-metrics")] /// Specify the port for the OpenMetrics server. /// @@ -195,6 +199,7 @@ fn main() -> Result<()> { bootstrap_peers, opt.local, root_dir, + opt.owner.clone(), ); node_builder.is_behind_home_network = opt.home_network; #[cfg(feature = "open-metrics")] diff --git a/sn_node/src/node.rs b/sn_node/src/node.rs index cc3e6a920f..ace0473d59 100644 --- a/sn_node/src/node.rs +++ b/sn_node/src/node.rs @@ -67,6 +67,7 @@ pub struct NodeBuilder { metrics_server_port: u16, /// Enable hole punching for nodes connecting from home networks. pub is_behind_home_network: bool, + owner: String, } impl NodeBuilder { @@ -77,6 +78,7 @@ impl NodeBuilder { initial_peers: Vec, local: bool, root_dir: PathBuf, + owner: String, ) -> Self { Self { keypair, @@ -87,6 +89,7 @@ impl NodeBuilder { #[cfg(feature = "open-metrics")] metrics_server_port: 0, is_behind_home_network: false, + owner, } } @@ -130,7 +133,8 @@ impl NodeBuilder { (metrics_registry, node_metrics) }; - let mut network_builder = NetworkBuilder::new(self.keypair, self.local, self.root_dir); + let mut network_builder = + NetworkBuilder::new(self.keypair, self.local, self.root_dir, self.owner.clone()); network_builder.listen_addr(self.addr); #[cfg(feature = "open-metrics")] diff --git a/sn_node/src/quote.rs b/sn_node/src/quote.rs index dbd5ed2a4c..a673e0afba 100644 --- a/sn_node/src/quote.rs +++ b/sn_node/src/quote.rs @@ -22,7 +22,13 @@ impl Node { ) -> Result { let content = address.as_xorname().unwrap_or_default(); let timestamp = std::time::SystemTime::now(); - let bytes = PaymentQuote::bytes_for_signing(content, cost, timestamp, quoting_metrics); + let bytes = PaymentQuote::bytes_for_signing( + content, + cost, + timestamp, + quoting_metrics, + network.owner(), + ); let Ok(signature) = network.sign(&bytes) else { return Err(ProtocolError::QuoteGenerationFailed); @@ -33,6 +39,7 @@ impl Node { cost, timestamp, quoting_metrics: quoting_metrics.clone(), + reason: network.owner(), pub_key: network.get_pub_key(), signature, }; @@ -65,6 +72,7 @@ pub(crate) fn verify_quote_for_storecost( quote.cost, quote.timestamp, "e.quoting_metrics, + quote.reason.clone(), ); let signature = quote.signature; if !network.verify(&bytes, &signature) { diff --git a/sn_node/tests/double_spend.rs b/sn_node/tests/double_spend.rs index 990eb0cff7..5aa86966bc 100644 --- a/sn_node/tests/double_spend.rs +++ b/sn_node/tests/double_spend.rs @@ -49,8 +49,18 @@ async fn cash_note_transfer_double_spend_fail() -> Result<()> { let mut rng = rng::thread_rng(); - let to2_unique_key = (amount, to2, DerivationIndex::random(&mut rng)); - let to3_unique_key = (amount, to3, DerivationIndex::random(&mut rng)); + let to2_unique_key = ( + amount, + Default::default(), + to2, + DerivationIndex::random(&mut rng), + ); + let to3_unique_key = ( + amount, + Default::default(), + to3, + DerivationIndex::random(&mut rng), + ); let reason_hash = Hash::default(); let transfer_to_2 = @@ -108,6 +118,7 @@ async fn genesis_double_spend_fail() -> Result<()> { let mut rng = rng::thread_rng(); let recipient = ( genesis_amount, + Default::default(), first_wallet_addr, DerivationIndex::random(&mut rng), ); @@ -131,6 +142,7 @@ async fn genesis_double_spend_fail() -> Result<()> { let (genesis_cashnote_and_others, exclusive_access) = first_wallet.available_cash_notes()?; let recipient = ( genesis_amount, + Default::default(), second_wallet_addr, DerivationIndex::random(&mut rng), ); diff --git a/sn_node_manager/src/bin/cli/main.rs b/sn_node_manager/src/bin/cli/main.rs index fd5b641bb3..a692fb83ef 100644 --- a/sn_node_manager/src/bin/cli/main.rs +++ b/sn_node_manager/src/bin/cli/main.rs @@ -547,6 +547,9 @@ pub enum LocalSubCmd { /// Set to skip the network validation process #[clap(long)] skip_validation: bool, + /// Specify the owner(readable discord user name). + #[clap(long)] + owner: Option, }, /// Run a local network. /// @@ -600,6 +603,8 @@ pub enum LocalSubCmd { /// Set to skip the network validation process #[clap(long)] skip_validation: bool, + #[clap(long)] + owner: Option, }, } @@ -712,13 +717,16 @@ async fn main() -> Result<()> { node_version, peers, skip_validation: _, + owner, } => { + let owner = owner.unwrap_or("maidsafe_test".to_string()); cmd::local::join( build, count, faucet_path, faucet_version, interval, + owner, node_path, node_version, peers, @@ -737,10 +745,13 @@ async fn main() -> Result<()> { node_path, node_version, skip_validation: _, + owner, } => { + let owner = owner.unwrap_or("maidsafe_test".to_string()); cmd::local::run( build, clean, + owner, count, faucet_path, faucet_version, diff --git a/sn_node_manager/src/cmd/local.rs b/sn_node_manager/src/cmd/local.rs index 3e211c6ae7..d3ee4baba9 100644 --- a/sn_node_manager/src/cmd/local.rs +++ b/sn_node_manager/src/cmd/local.rs @@ -27,6 +27,7 @@ pub async fn join( faucet_path: Option, faucet_version: Option, interval: u64, + owner: String, node_path: Option, node_version: Option, peers: PeersArgs, @@ -68,6 +69,7 @@ pub async fn join( }; let options = LocalNetworkOptions { faucet_bin_path: faucet_path, + owner, interval, join: true, node_count: count, @@ -99,6 +101,7 @@ pub fn kill(keep_directories: bool, verbosity: VerbosityLevel) -> Result<()> { pub async fn run( build: bool, clean: bool, + owner: String, count: u16, faucet_path: Option, faucet_version: Option, @@ -156,6 +159,7 @@ pub async fn run( let options = LocalNetworkOptions { faucet_bin_path: faucet_path, + owner, join: false, interval, node_count: count, diff --git a/sn_node_manager/src/local.rs b/sn_node_manager/src/local.rs index 3d270c84e2..f127f2c728 100644 --- a/sn_node_manager/src/local.rs +++ b/sn_node_manager/src/local.rs @@ -33,6 +33,7 @@ pub trait Launcher { fn launch_faucet(&self, genesis_multiaddr: &Multiaddr) -> Result; fn launch_node( &self, + owner: String, rpc_socket_addr: SocketAddr, bootstrap_peers: Vec, ) -> Result<()>; @@ -66,10 +67,15 @@ impl Launcher for LocalSafeLauncher { fn launch_node( &self, + owner: String, rpc_socket_addr: SocketAddr, bootstrap_peers: Vec, ) -> Result<()> { let mut args = Vec::new(); + + args.push("--owner".to_string()); + args.push(owner); + if bootstrap_peers.is_empty() { args.push("--first".to_string()) } else { @@ -162,6 +168,7 @@ pub fn kill_network(node_registry: &NodeRegistry, keep_directories: bool) -> Res pub struct LocalNetworkOptions { pub faucet_bin_path: PathBuf, + pub owner: String, pub join: bool, pub interval: u64, pub node_count: u16, @@ -200,6 +207,7 @@ pub async fn run_network( let node = run_node( RunNodeOptions { version: get_bin_version(&launcher.get_safenode_path())?, + owner: options.owner.clone(), number, genesis: true, interval: options.interval, @@ -227,6 +235,7 @@ pub async fn run_network( let node = run_node( RunNodeOptions { version: get_bin_version(&launcher.get_safenode_path())?, + owner: options.owner.clone(), number, genesis: false, interval: options.interval, @@ -274,6 +283,7 @@ pub async fn run_network( pub struct RunNodeOptions { pub version: String, + pub owner: String, pub number: u16, pub genesis: bool, pub interval: u64, @@ -286,8 +296,31 @@ pub async fn run_node( launcher: &dyn Launcher, rpc_client: &dyn RpcActions, ) -> Result { + let user = match get_username() { + Ok(user_name) => { + println!("parsed env set user_name is {user_name:?}"); + if user_name.is_empty() { + println!( + "Env set user_name is empty, using owner ({:?}) passed via cli.", + run_options.owner + ); + run_options.owner.clone() + } else { + user_name + } + } + Err(_err) => { + println!( + "cannot parse user_name from env, using owner ({:?}) passed via cli.", + run_options.owner + ); + run_options.owner.clone() + } + }; + println!("Launching node {}...", run_options.number); launcher.launch_node( + user.clone(), run_options.rpc_socket_addr, run_options.bootstrap_peers.clone(), )?; @@ -309,7 +342,7 @@ pub async fn run_node( home_network: false, local: true, service_name: format!("safenode-local{}", run_options.number), - user: get_username()?, + user, number: run_options.number, rpc_socket_addr: run_options.rpc_socket_addr, version: run_options.version.to_string(), @@ -412,14 +445,15 @@ mod tests { async fn run_node_should_launch_the_genesis_node() -> Result<()> { let mut mock_launcher = MockLauncher::new(); let mut mock_rpc_client = MockRpcClient::new(); + let owner = get_username()?; let peer_id = PeerId::from_str("12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR")?; let rpc_socket_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 13000); mock_launcher .expect_launch_node() - .with(eq(rpc_socket_addr), eq(vec![])) + .with(eq(owner.clone()), eq(rpc_socket_addr), eq(vec![])) .times(1) - .returning(|_, _| Ok(())); + .returning(|_, _, _| Ok(())); mock_launcher .expect_wait() .with(eq(100)) @@ -456,6 +490,7 @@ mod tests { let node = run_node( RunNodeOptions { version: "0.100.12".to_string(), + owner, number: 1, genesis: true, interval: 100, diff --git a/sn_transfers/benches/reissue.rs b/sn_transfers/benches/reissue.rs index b2c8cfc900..17e3805fe9 100644 --- a/sn_transfers/benches/reissue.rs +++ b/sn_transfers/benches/reissue.rs @@ -26,6 +26,7 @@ fn bench_reissue_1_to_100(c: &mut Criterion) { let main_key = MainSecretKey::random_from_rng(&mut rng); ( NanoTokens::from(1), + Default::default(), main_key.main_pubkey(), DerivationIndex::random(&mut rng), ) @@ -81,6 +82,7 @@ fn bench_reissue_100_to_1(c: &mut Criterion) { .map(|_| { ( NanoTokens::from(1), + Default::default(), recipient_of_100_mainkey.main_pubkey(), DerivationIndex::random(&mut rng), ) @@ -130,6 +132,7 @@ fn bench_reissue_100_to_1(c: &mut Criterion) { .collect(); let one_single_recipient = vec![( NanoTokens::from(total_amount), + Default::default(), starting_main_key.main_pubkey(), DerivationIndex::random(&mut rng), )]; diff --git a/sn_transfers/src/cashnotes.rs b/sn_transfers/src/cashnotes.rs index 9920a0f2bd..0448054b74 100644 --- a/sn_transfers/src/cashnotes.rs +++ b/sn_transfers/src/cashnotes.rs @@ -20,7 +20,7 @@ pub(crate) use transaction::Input; pub use address::SpendAddress; pub use builder::UnsignedTransfer; -pub use cashnote::CashNote; +pub use cashnote::{CashNote, CashNoteOutputDetails}; pub use nano::NanoTokens; pub use reason_hash::Hash; pub use signed_spend::{SignedSpend, Spend}; @@ -49,6 +49,7 @@ pub(crate) mod tests { unique_pubkey: derived_key.unique_pubkey(), parent_tx: tx, parent_spends: Default::default(), + reason: Default::default(), main_pubkey: main_key.main_pubkey(), derivation_index, }; @@ -76,6 +77,7 @@ pub(crate) mod tests { unique_pubkey: derived_key.unique_pubkey(), parent_tx: tx, parent_spends: Default::default(), + reason: Default::default(), main_pubkey: main_key.main_pubkey(), derivation_index, }; @@ -107,6 +109,7 @@ pub(crate) mod tests { unique_pubkey: derived_key.unique_pubkey(), parent_tx: tx, parent_spends: Default::default(), + reason: Default::default(), main_pubkey: main_key.main_pubkey(), derivation_index, }; @@ -138,6 +141,7 @@ pub(crate) mod tests { unique_pubkey: derived_key.unique_pubkey(), parent_tx: tx, parent_spends: Default::default(), + reason: Default::default(), main_pubkey: main_key.main_pubkey(), derivation_index, }; diff --git a/sn_transfers/src/cashnotes/builder.rs b/sn_transfers/src/cashnotes/builder.rs index d43d86d692..ab2a6b255b 100644 --- a/sn_transfers/src/cashnotes/builder.rs +++ b/sn_transfers/src/cashnotes/builder.rs @@ -8,11 +8,11 @@ use super::{ transaction::{Output, Transaction}, - CashNote, DerivationIndex, DerivedSecretKey, Hash, Input, MainPubkey, NanoTokens, SignedSpend, - Spend, UniquePubkey, + CashNote, CashNoteOutputDetails, DerivationIndex, DerivedSecretKey, Hash, Input, MainPubkey, + NanoTokens, SignedSpend, Spend, UniquePubkey, }; -use crate::{Result, TransferError}; +use crate::{transfers::TransferRecipientDetails, Result, TransferError}; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; @@ -31,7 +31,7 @@ pub struct UnsignedTransfer { /// spending the necessary input cash_notes. pub change_id: UniquePubkey, /// Information for aggregating signed spends and generating the final CashNote outputs. - pub output_details: BTreeMap, + pub output_details: BTreeMap, } /// A builder to create a Transaction from @@ -41,7 +41,7 @@ pub struct TransactionBuilder { inputs: Vec, outputs: Vec, input_details: BTreeMap, InputSrcTx, DerivationIndex)>, - output_details: BTreeMap, + output_details: BTreeMap, } impl TransactionBuilder { @@ -72,17 +72,18 @@ impl TransactionBuilder { self } - /// Add an output given the token, the MainPubkey and the DerivationIndex + /// Add an output given the token, reason, the MainPubkey and the DerivationIndex pub fn add_output( mut self, token: NanoTokens, + reason: String, main_pubkey: MainPubkey, derivation_index: DerivationIndex, ) -> Self { let unique_pubkey = main_pubkey.new_unique_pubkey(&derivation_index); self.output_details - .insert(unique_pubkey, (main_pubkey, derivation_index)); + .insert(unique_pubkey, (reason, main_pubkey, derivation_index)); let output = Output::new(unique_pubkey, token.as_nano()); self.outputs.push(output); @@ -92,10 +93,10 @@ impl TransactionBuilder { /// Add a list of outputs given the tokens, the MainPubkey and the DerivationIndex pub fn add_outputs( mut self, - outputs: impl IntoIterator, + outputs: impl IntoIterator, ) -> Self { - for (token, main_pubkey, derivation_index) in outputs.into_iter() { - self = self.add_output(token, main_pubkey, derivation_index); + for (token, reason, main_pubkey, derivation_index) in outputs.into_iter() { + self = self.add_output(token, reason, main_pubkey, derivation_index); } self } @@ -179,7 +180,7 @@ impl TransactionBuilder { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CashNoteBuilder { pub spent_tx: Transaction, - pub output_details: BTreeMap, + pub output_details: BTreeMap, pub signed_spends: BTreeSet, } @@ -187,7 +188,7 @@ impl CashNoteBuilder { /// Create a new CashNoteBuilder. pub fn new( spent_tx: Transaction, - output_details: BTreeMap, + output_details: BTreeMap, signed_spends: BTreeSet, ) -> Self { Self { @@ -228,7 +229,7 @@ impl CashNoteBuilder { .outputs .iter() .map(|output| { - let (main_pubkey, derivation_index) = self + let (reason, main_pubkey, derivation_index) = self .output_details .get(&output.unique_pubkey) .ok_or(TransferError::UniquePubkeyNotFound)?; @@ -238,6 +239,7 @@ impl CashNoteBuilder { unique_pubkey: main_pubkey.new_unique_pubkey(derivation_index), parent_tx: self.spent_tx.clone(), parent_spends: self.signed_spends.clone(), + reason: reason.clone(), main_pubkey: *main_pubkey, derivation_index: *derivation_index, }, diff --git a/sn_transfers/src/cashnotes/cashnote.rs b/sn_transfers/src/cashnotes/cashnote.rs index d14a5886de..84fcd00dda 100644 --- a/sn_transfers/src/cashnotes/cashnote.rs +++ b/sn_transfers/src/cashnotes/cashnote.rs @@ -17,6 +17,9 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeSet; use tiny_keccak::{Hasher, Sha3}; +/// OutputDetails: (CashNoteReason, pub_key, derived_key) +pub type CashNoteOutputDetails = (String, MainPubkey, DerivationIndex); + /// Represents a CashNote (CashNote). /// /// A CashNote is like a check. Only the recipient can spend it. @@ -66,6 +69,9 @@ pub struct CashNote { pub parent_tx: Transaction, /// The transaction's input's SignedSpends pub parent_spends: BTreeSet, + /// The reason this cash_note created for + /// eg. `store cost pay to...`, `network royalty`, `change to self`, `payment transfer`, ... + pub reason: String, /// This is the MainPubkey of the owner of this CashNote pub main_pubkey: MainPubkey, /// The derivation index used to derive the UniquePubkey and DerivedSecretKey from the MainPubkey and MainSecretKey respectively. @@ -112,7 +118,7 @@ impl CashNote { /// Return the reason why this CashNote was spent. /// Will be the default Hash (empty) if reason is none. - pub fn reason(&self) -> Hash { + pub fn spent_reason(&self) -> Hash { self.parent_spends .iter() .next() @@ -120,6 +126,11 @@ impl CashNote { .unwrap_or_default() } + /// Return the reason why this CashNote was created. + pub fn self_reason(&self) -> String { + self.reason.clone() + } + /// Return the value in NanoTokens for this CashNote. pub fn value(&self) -> Result { Ok(self @@ -142,7 +153,9 @@ impl CashNote { sha3.update(&sp.to_bytes()); } - sha3.update(self.reason().as_ref()); + sha3.update(&self.reason.clone().into_bytes()); + + sha3.update(self.spent_reason().as_ref()); let mut hash = [0u8; 32]; sha3.finalize(&mut hash); Hash::from(hash) @@ -176,8 +189,8 @@ impl CashNote { } // verify that all signed_spend reasons are equal - let reason = self.reason(); - let reasons_are_equal = |s: &SignedSpend| reason == s.reason(); + let spent_reason = self.spent_reason(); + let reasons_are_equal = |s: &SignedSpend| spent_reason == s.reason(); if !self.parent_spends.iter().all(reasons_are_equal) { return Err(TransferError::SignedSpendReasonMismatch(unique_pubkey)); } diff --git a/sn_transfers/src/genesis.rs b/sn_transfers/src/genesis.rs index 4d590957dd..2b02b68049 100644 --- a/sn_transfers/src/genesis.rs +++ b/sn_transfers/src/genesis.rs @@ -148,7 +148,7 @@ pub fn create_first_cash_note_from_key( amount: NanoTokens::from(GENESIS_CASHNOTE_AMOUNT), }; - let reason = Hash::hash(b"GENESIS"); + let reason = Hash::hash(b"SPEND_REASON_FOR_GENESIS"); let cash_note_builder = TransactionBuilder::default() .add_input( @@ -159,6 +159,7 @@ pub fn create_first_cash_note_from_key( ) .add_output( NanoTokens::from(GENESIS_CASHNOTE_AMOUNT), + "CASH_NOTE_REASON_FOR_GENESIS".to_string(), main_pubkey, derivation_index, ) diff --git a/sn_transfers/src/lib.rs b/sn_transfers/src/lib.rs index 449247fc70..90dc0cf21e 100644 --- a/sn_transfers/src/lib.rs +++ b/sn_transfers/src/lib.rs @@ -19,8 +19,9 @@ pub(crate) use cashnotes::{Input, TransactionBuilder}; /// Types used in the public API pub use cashnotes::{ - CashNote, DerivationIndex, DerivedSecretKey, Hash, MainPubkey, MainSecretKey, NanoTokens, - SignedSpend, Spend, SpendAddress, Transaction, UniquePubkey, UnsignedTransfer, + CashNote, CashNoteOutputDetails, DerivationIndex, DerivedSecretKey, Hash, MainPubkey, + MainSecretKey, NanoTokens, SignedSpend, Spend, SpendAddress, Transaction, UniquePubkey, + UnsignedTransfer, }; pub use error::{Result, TransferError}; pub use transfers::{CashNoteRedemption, OfflineTransfer, Transfer}; diff --git a/sn_transfers/src/transfers.rs b/sn_transfers/src/transfers.rs index 545ce9eaa8..606ee9535f 100644 --- a/sn_transfers/src/transfers.rs +++ b/sn_transfers/src/transfers.rs @@ -30,5 +30,7 @@ mod offline_transfer; mod transfer; -pub use offline_transfer::{create_unsigned_transfer, CashNotesAndSecretKey, OfflineTransfer}; +pub use offline_transfer::{ + create_unsigned_transfer, CashNotesAndSecretKey, OfflineTransfer, TransferRecipientDetails, +}; pub use transfer::{CashNoteRedemption, Transfer}; diff --git a/sn_transfers/src/transfers/offline_transfer.rs b/sn_transfers/src/transfers/offline_transfer.rs index 5a43a1fc96..bd2c6b32f6 100644 --- a/sn_transfers/src/transfers/offline_transfer.rs +++ b/sn_transfers/src/transfers/offline_transfer.rs @@ -8,9 +8,9 @@ use crate::{ cashnotes::{CashNoteBuilder, UnsignedTransfer}, - rng, CashNote, DerivationIndex, DerivedSecretKey, Hash, Input, MainPubkey, NanoTokens, Result, - SignedSpend, Transaction, TransactionBuilder, TransferError, UniquePubkey, - NETWORK_ROYALTIES_PK, + rng, CashNote, CashNoteOutputDetails, DerivationIndex, DerivedSecretKey, Hash, Input, + MainPubkey, NanoTokens, Result, SignedSpend, Transaction, TransactionBuilder, TransferError, + UniquePubkey, NETWORK_ROYALTIES_PK, }; use serde::{Deserialize, Serialize}; @@ -19,6 +19,9 @@ use std::collections::{BTreeMap, BTreeSet}; /// List of CashNotes, with (optionally when needed) their corresponding derived owning secret key. pub type CashNotesAndSecretKey = Vec<(CashNote, Option)>; +/// RecipientDetails: (amount, cash_note_reason, pub_key, derivation_index) +pub type TransferRecipientDetails = (NanoTokens, String, MainPubkey, DerivationIndex); + /// Offline Transfer /// This struct contains all the necessary information to carry out the transfer. /// The created cash_notes and change cash_note from a transfer @@ -45,7 +48,7 @@ impl OfflineTransfer { signed_spends: BTreeSet, tx: Transaction, change_id: UniquePubkey, - output_details: BTreeMap, + output_details: BTreeMap, ) -> Result { let cash_note_builder = CashNoteBuilder::new(tx.clone(), output_details, signed_spends.clone()); @@ -88,13 +91,13 @@ impl OfflineTransfer { /// them upon request, the transaction will be completed. pub fn new( available_cash_notes: CashNotesAndSecretKey, - recipients: Vec<(NanoTokens, MainPubkey, DerivationIndex)>, + recipients: Vec<(NanoTokens, String, MainPubkey, DerivationIndex)>, change_to: MainPubkey, reason_hash: Hash, ) -> Result { let total_output_amount = recipients .iter() - .try_fold(NanoTokens::zero(), |total, (amount, _, _)| { + .try_fold(NanoTokens::zero(), |total, (amount, _, _, _)| { total.checked_add(*amount) }) .ok_or_else(|| { @@ -125,7 +128,7 @@ struct TransferInputs { /// to transfer the below specified amount of tokens to each recipients. pub cash_notes_to_spend: CashNotesAndSecretKey, /// The amounts and cash_note ids for the cash_notes that will be created to hold the transferred tokens. - pub recipients: Vec<(NanoTokens, MainPubkey, DerivationIndex)>, + pub recipients: Vec<(NanoTokens, String, MainPubkey, DerivationIndex)>, /// Any surplus amount after spending the necessary input cash_notes. pub change: (NanoTokens, MainPubkey), } @@ -133,13 +136,13 @@ struct TransferInputs { /// A function for creating an unsigned transfer of tokens. pub fn create_unsigned_transfer( available_cash_notes: CashNotesAndSecretKey, - recipients: Vec<(NanoTokens, MainPubkey, DerivationIndex)>, + recipients: Vec<(NanoTokens, String, MainPubkey, DerivationIndex)>, change_to: MainPubkey, reason_hash: Hash, ) -> Result { let total_output_amount = recipients .iter() - .try_fold(NanoTokens::zero(), |total, (amount, _, _)| { + .try_fold(NanoTokens::zero(), |total, (amount, _, _, _)| { total.checked_add(*amount) }) .ok_or(TransferError::ExcessiveNanoValue)?; @@ -158,8 +161,8 @@ pub fn create_unsigned_transfer( let network_royalties: Vec = selected_inputs .recipients .iter() - .filter(|(_, main_pubkey, _)| *main_pubkey == *NETWORK_ROYALTIES_PK) - .map(|(_, _, derivation_index)| *derivation_index) + .filter(|(_, _, main_pubkey, _)| *main_pubkey == *NETWORK_ROYALTIES_PK) + .map(|(_, _, _, derivation_index)| *derivation_index) .collect(); let (tx_builder, _src_txs, change_id) = create_transaction_builder_with(selected_inputs)?; @@ -272,7 +275,12 @@ fn create_transaction_builder_with( let derivation_index = DerivationIndex::random(&mut rng); let change_id = change_to.new_unique_pubkey(&derivation_index); if !change.is_zero() { - tx_builder = tx_builder.add_output(change, change_to, derivation_index); + tx_builder = tx_builder.add_output( + change, + "CASH_NOTE_REASON_FOR_CHANGE".to_string(), + change_to, + derivation_index, + ); } Ok((tx_builder, src_txs, change_id)) @@ -292,8 +300,8 @@ fn create_offline_transfer_with( let network_royalties: Vec = selected_inputs .recipients .iter() - .filter(|(_, main_pubkey, _)| *main_pubkey == *NETWORK_ROYALTIES_PK) - .map(|(_, _, derivation_index)| *derivation_index) + .filter(|(_, _, main_pubkey, _)| *main_pubkey == *NETWORK_ROYALTIES_PK) + .map(|(_, _, _, derivation_index)| *derivation_index) .collect(); let (tx_builder, src_txs, change_id) = create_transaction_builder_with(selected_inputs)?; diff --git a/sn_transfers/src/transfers/transfer.rs b/sn_transfers/src/transfers/transfer.rs index 89f06c0633..514ed716de 100644 --- a/sn_transfers/src/transfers/transfer.rs +++ b/sn_transfers/src/transfers/transfer.rs @@ -138,14 +138,21 @@ pub struct CashNoteRedemption { /// spentbook entry of one of one of the inputs (parent spends) /// using data found at this address the owner can check that the output is valid money pub parent_spend: SpendAddress, + /// For what purpose this cash_note was created + pub reason: String, } impl CashNoteRedemption { /// Create a new CashNoteRedemption - pub fn new(derivation_index: DerivationIndex, parent_spend: SpendAddress) -> Self { + pub fn new( + derivation_index: DerivationIndex, + parent_spend: SpendAddress, + reason: String, + ) -> Self { Self { derivation_index, parent_spend, + reason, } } @@ -157,7 +164,11 @@ impl CashNoteRedemption { return Err(TransferError::CashNoteHasNoParentSpends); } }; - Ok(Self::new(derivation_index, parent_spend)) + Ok(Self::new( + derivation_index, + parent_spend, + cash_note.reason.clone(), + )) } /// Serialize the CashNoteRedemption to bytes @@ -199,6 +210,7 @@ mod tests { let cashnote_redemption = CashNoteRedemption::new( DerivationIndex([42; 32]), SpendAddress::new(XorName::random(rng)), + Default::default(), ); let sk = MainSecretKey::random(); let pk = sk.main_pubkey(); @@ -219,6 +231,7 @@ mod tests { let cashnote_redemption = CashNoteRedemption::new( DerivationIndex([42; 32]), SpendAddress::new(XorName::random(rng)), + Default::default(), ); let sk = MainSecretKey::random(); let pk = sk.main_pubkey(); diff --git a/sn_transfers/src/wallet/data_payments.rs b/sn_transfers/src/wallet/data_payments.rs index fcaa05fc44..b7e0164522 100644 --- a/sn_transfers/src/wallet/data_payments.rs +++ b/sn_transfers/src/wallet/data_payments.rs @@ -106,6 +106,8 @@ pub struct PaymentQuote { pub timestamp: SystemTime, /// quoting metrics being used to generate this quote pub quoting_metrics: QuotingMetrics, + /// node's reason to accept the payment. Normally using its discord username + pub reason: String, /// node's public key that can verify the signature #[debug(skip)] pub pub_key: Vec, @@ -121,6 +123,7 @@ impl PaymentQuote { cost: NanoTokens::zero(), timestamp: SystemTime::now(), quoting_metrics: Default::default(), + reason: Default::default(), pub_key: vec![], signature: vec![], } @@ -132,6 +135,7 @@ impl PaymentQuote { cost: NanoTokens, timestamp: SystemTime, quoting_metrics: &QuotingMetrics, + reason: String, ) -> Vec { let mut bytes = xorname.to_vec(); bytes.extend_from_slice(&cost.to_bytes()); @@ -147,6 +151,7 @@ impl PaymentQuote { Err(_err) => vec![], }; bytes.extend_from_slice(&serialised_quoting_metrics); + bytes.extend_from_slice(&reason.into_bytes()); bytes } @@ -171,6 +176,7 @@ impl PaymentQuote { self.cost, self.timestamp, &self.quoting_metrics, + self.reason.clone(), ); if !pub_key.verify(&bytes, &self.signature) { @@ -199,6 +205,7 @@ impl PaymentQuote { cost, timestamp: SystemTime::now(), quoting_metrics: Default::default(), + reason: Default::default(), pub_key: vec![], signature: vec![], } @@ -297,6 +304,7 @@ mod tests { quote.cost, quote.timestamp, "e.quoting_metrics, + quote.reason.clone(), ); let signature = if let Ok(sig) = keypair.sign(&bytes) { sig diff --git a/sn_transfers/src/wallet/hot_wallet.rs b/sn_transfers/src/wallet/hot_wallet.rs index 9e15f59679..563d601fbb 100644 --- a/sn_transfers/src/wallet/hot_wallet.rs +++ b/sn_transfers/src/wallet/hot_wallet.rs @@ -22,8 +22,8 @@ use crate::{ calculate_royalties_fee, cashnotes::UnsignedTransfer, transfers::{CashNotesAndSecretKey, OfflineTransfer}, - CashNote, CashNoteRedemption, DerivationIndex, DerivedSecretKey, Hash, MainPubkey, - MainSecretKey, NanoTokens, SignedSpend, Spend, Transaction, Transfer, UniquePubkey, + CashNote, CashNoteOutputDetails, CashNoteRedemption, DerivationIndex, DerivedSecretKey, Hash, + MainPubkey, MainSecretKey, NanoTokens, SignedSpend, Spend, Transaction, Transfer, UniquePubkey, WalletError, NETWORK_ROYALTIES_PK, }; use std::{ @@ -37,6 +37,9 @@ use xor_name::XorName; /// A locked file handle, that when dropped releases the lock. pub type WalletExclusiveAccess = File; +// TransactionPayeeDetails: (reason, amount, address) +pub type TransactionPayeeDetails = (String, NanoTokens, MainPubkey); + /// A hot-wallet. pub struct HotWallet { /// The secret key with which we can access @@ -303,7 +306,7 @@ impl HotWallet { pub fn build_unsigned_transaction( &mut self, - to: Vec<(NanoTokens, MainPubkey)>, + to: Vec, reason_hash: Option, ) -> Result { self.watchonly_wallet @@ -313,14 +316,16 @@ impl HotWallet { /// Make a transfer and return all created cash_notes pub fn local_send( &mut self, - to: Vec<(NanoTokens, MainPubkey)>, + to: Vec, reason_hash: Option, ) -> Result> { let mut rng = &mut rand::rngs::OsRng; // create a unique key for each output let to_unique_keys: Vec<_> = to .into_iter() - .map(|(amount, address)| (amount, address, DerivationIndex::random(&mut rng))) + .map(|(reason, amount, address)| { + (amount, reason, address, DerivationIndex::random(&mut rng)) + }) .collect(); let (available_cash_notes, exclusive_access) = self.available_cash_notes()?; @@ -352,7 +357,7 @@ impl HotWallet { signed_spends: BTreeSet, tx: Transaction, change_id: UniquePubkey, - output_details: BTreeMap, + output_details: BTreeMap, ) -> Result> { let transfer = OfflineTransfer::from_transaction(signed_spends, tx, change_id, output_details)?; @@ -389,6 +394,7 @@ impl HotWallet { for (xorname, (main_pubkey, quote, peer_id_bytes)) in price_map.iter() { let storage_payee = ( quote.cost, + quote.reason.clone(), *main_pubkey, DerivationIndex::random(&mut rng), peer_id_bytes.clone(), @@ -396,6 +402,7 @@ impl HotWallet { let royalties_fee = calculate_royalties_fee(quote.cost); let royalties_payee = ( royalties_fee, + "CASH_NOTE_REASON_FOR_NETWORK_ROYALTIES".to_string(), *NETWORK_ROYALTIES_PK, DerivationIndex::random(&mut rng), ); @@ -413,7 +420,14 @@ impl HotWallet { // create offline transfers let recipients = recipients_by_xor .values() - .flat_map(|(node, roy)| vec![(node.0, node.1, node.2), *roy]) + .flat_map( + |((cost, reason, pubkey, derivation_index, _id_bytes), roy)| { + vec![ + (*cost, reason.clone(), *pubkey, *derivation_index), + roy.clone(), + ] + }, + ) .collect(); trace!( @@ -429,7 +443,7 @@ impl HotWallet { start.elapsed() ); debug!("Available CashNotes: {:#?}", available_cash_notes); - let reason_hash = Default::default(); + let reason_hash = Hash::hash(b"SPEND_REASON_FOR_STORAGE"); let start = Instant::now(); let offline_transfer = OfflineTransfer::new( available_cash_notes, @@ -452,7 +466,7 @@ impl HotWallet { .collect(); for (xorname, recipients_info) in recipients_by_xor { let (storage_payee, royalties_payee) = recipients_info; - let (pay_amount, node_key, _, peer_id_bytes) = storage_payee; + let (pay_amount, _reason, node_key, _, peer_id_bytes) = storage_payee; let cash_note_for_node = cashnotes_to_use .iter() .find(|cash_note| { @@ -467,7 +481,7 @@ impl HotWallet { let transfer_for_node = Transfer::transfer_from_cash_note(&cash_note_for_node)?; trace!("Created transaction regarding {xorname:?} paying {transfer_amount:?} to {node_key:?}."); - let royalties_key = royalties_payee.1; + let royalties_key = royalties_payee.2; let royalties_amount = royalties_payee.0; let cash_note_for_royalties = cashnotes_to_use .iter() @@ -856,7 +870,11 @@ mod tests { let send_amount = 100; let recipient_key = MainSecretKey::random(); let recipient_main_pubkey = recipient_key.main_pubkey(); - let to = vec![(NanoTokens::from(send_amount), recipient_main_pubkey)]; + let to = vec![( + Default::default(), + NanoTokens::from(send_amount), + recipient_main_pubkey, + )]; let created_cash_notes = sender.local_send(to, None)?; assert_eq!(1, created_cash_notes.len()); @@ -888,7 +906,11 @@ mod tests { let send_amount = 100; let recipient_key = MainSecretKey::random(); let recipient_main_pubkey = recipient_key.main_pubkey(); - let to = vec![(NanoTokens::from(send_amount), recipient_main_pubkey)]; + let to = vec![( + Default::default(), + NanoTokens::from(send_amount), + recipient_main_pubkey, + )]; let _created_cash_notes = sender.local_send(to, None)?; let deserialized = HotWallet::load_from(&root_dir)?; @@ -949,7 +971,11 @@ mod tests { let recipient_main_pubkey = recipient.key.main_pubkey(); - let to = vec![(NanoTokens::from(send_amount), recipient_main_pubkey)]; + let to = vec![( + Default::default(), + NanoTokens::from(send_amount), + recipient_main_pubkey, + )]; let created_cash_notes = sender.local_send(to, None)?; let cash_note = created_cash_notes[0].clone(); let unique_pubkey = cash_note.unique_pubkey(); diff --git a/sn_transfers/src/wallet/watch_only.rs b/sn_transfers/src/wallet/watch_only.rs index 8cfdc58923..bea31b9aaf 100644 --- a/sn_transfers/src/wallet/watch_only.rs +++ b/sn_transfers/src/wallet/watch_only.rs @@ -9,7 +9,7 @@ use super::{ api::WalletApi, error::{Error, Result}, - hot_wallet::WalletExclusiveAccess, + hot_wallet::{TransactionPayeeDetails, WalletExclusiveAccess}, keys::{get_main_pubkey, store_new_pubkey}, wallet_file::{ load_cash_notes_from_disk, load_created_cash_note, store_created_cash_notes, store_wallet, @@ -215,14 +215,16 @@ impl WatchOnlyWallet { pub fn build_unsigned_transaction( &mut self, - to: Vec<(NanoTokens, MainPubkey)>, + to: Vec, reason_hash: Option, ) -> Result { let mut rng = &mut rand::rngs::OsRng; // create a unique key for each output let to_unique_keys: Vec<_> = to .into_iter() - .map(|(amount, address)| (amount, address, DerivationIndex::random(&mut rng))) + .map(|(reason, amount, address)| { + (amount, reason, address, DerivationIndex::random(&mut rng)) + }) .collect(); trace!("Trying to lock wallet to get available cash_notes..."); From e8d592ba147b92a6d03f3abdbd6b45a8b7f88fa4 Mon Sep 17 00:00:00 2001 From: qima Date: Mon, 29 Apr 2024 22:07:35 +0800 Subject: [PATCH 171/205] feat: spend shows the purposes of outputs created for --- sn_node/src/put_validation.rs | 9 ++++++++ sn_transfers/src/cashnotes.rs | 24 ++++++++++++++++++---- sn_transfers/src/cashnotes/builder.rs | 8 +++++--- sn_transfers/src/cashnotes/signed_spend.rs | 9 ++++++++ sn_transfers/src/cashnotes/transaction.rs | 8 +++++++- 5 files changed, 50 insertions(+), 8 deletions(-) diff --git a/sn_node/src/put_validation.rs b/sn_node/src/put_validation.rs index 4199f71df8..63e050bf73 100644 --- a/sn_node/src/put_validation.rs +++ b/sn_node/src/put_validation.rs @@ -81,6 +81,15 @@ impl Node { let record_key = record.key.clone(); let value_to_hash = record.value.clone(); let spends = try_deserialize_record::>(&record)?; + + let pretty_key = PrettyPrintRecordKey::from(&record_key); + for spend in spends.iter() { + trace!( + "Spend record {pretty_key:?} has output purposes of {:?}", + spend.spend.output_purposes() + ); + } + let result = self.validate_and_store_spends(spends, &record_key).await; if result.is_ok() { Marker::ValidSpendPutFromClient(&PrettyPrintRecordKey::from(&record_key)).log(); diff --git a/sn_transfers/src/cashnotes.rs b/sn_transfers/src/cashnotes.rs index 0448054b74..428e329d23 100644 --- a/sn_transfers/src/cashnotes.rs +++ b/sn_transfers/src/cashnotes.rs @@ -43,7 +43,11 @@ pub(crate) mod tests { let derived_key = main_key.derive_key(&derivation_index); let tx = Transaction { inputs: vec![], - outputs: vec![Output::new(derived_key.unique_pubkey(), amount)], + outputs: vec![Output::new( + derived_key.unique_pubkey(), + amount, + "maidsafe_test".to_string(), + )], }; let cashnote = CashNote { unique_pubkey: derived_key.unique_pubkey(), @@ -71,7 +75,11 @@ pub(crate) mod tests { let derived_key = main_key.derive_key(&derivation_index); let tx = Transaction { inputs: vec![], - outputs: vec![Output::new(derived_key.unique_pubkey(), amount)], + outputs: vec![Output::new( + derived_key.unique_pubkey(), + amount, + "maidsafe_test".to_string(), + )], }; let cashnote = CashNote { unique_pubkey: derived_key.unique_pubkey(), @@ -102,7 +110,11 @@ pub(crate) mod tests { let tx = Transaction { inputs: vec![], - outputs: vec![Output::new(derived_key.unique_pubkey(), amount)], + outputs: vec![Output::new( + derived_key.unique_pubkey(), + amount, + "maidsafe_test".to_string(), + )], }; let cashnote = CashNote { @@ -134,7 +146,11 @@ pub(crate) mod tests { let tx = Transaction { inputs: vec![], - outputs: vec![Output::new(derived_key.unique_pubkey(), amount)], + outputs: vec![Output::new( + derived_key.unique_pubkey(), + amount, + "maidsafe_test".to_string(), + )], }; let cashnote = CashNote { diff --git a/sn_transfers/src/cashnotes/builder.rs b/sn_transfers/src/cashnotes/builder.rs index ab2a6b255b..471190c8ab 100644 --- a/sn_transfers/src/cashnotes/builder.rs +++ b/sn_transfers/src/cashnotes/builder.rs @@ -82,9 +82,11 @@ impl TransactionBuilder { ) -> Self { let unique_pubkey = main_pubkey.new_unique_pubkey(&derivation_index); - self.output_details - .insert(unique_pubkey, (reason, main_pubkey, derivation_index)); - let output = Output::new(unique_pubkey, token.as_nano()); + self.output_details.insert( + unique_pubkey, + (reason.clone(), main_pubkey, derivation_index), + ); + let output = Output::new(unique_pubkey, token.as_nano(), reason); self.outputs.push(output); self diff --git a/sn_transfers/src/cashnotes/signed_spend.rs b/sn_transfers/src/cashnotes/signed_spend.rs index 174060d202..d0b71f4a5b 100644 --- a/sn_transfers/src/cashnotes/signed_spend.rs +++ b/sn_transfers/src/cashnotes/signed_spend.rs @@ -240,6 +240,15 @@ impl Spend { pub fn hash(&self) -> Hash { Hash::hash(&self.to_bytes()) } + + /// Returns the purpose of the outputs that associate with this Spend + pub fn output_purposes(&self) -> Vec<(String, NanoTokens)> { + self.spent_tx + .outputs + .iter() + .map(|output| (output.purpose.clone(), output.amount)) + .collect() + } } impl PartialOrd for Spend { diff --git a/sn_transfers/src/cashnotes/transaction.rs b/sn_transfers/src/cashnotes/transaction.rs index 19e40b9790..03b0b86954 100644 --- a/sn_transfers/src/cashnotes/transaction.rs +++ b/sn_transfers/src/cashnotes/transaction.rs @@ -15,6 +15,7 @@ type Result = std::result::Result; #[derive(Eq, PartialEq, Debug, Clone, Serialize, Deserialize, Hash)] pub struct Input { + // pub_key of the cash_note that as this input pub unique_pubkey: UniquePubkey, pub amount: NanoTokens, } @@ -41,15 +42,19 @@ impl Input { #[derive(Debug, Clone, Serialize, Deserialize, Hash, PartialEq, Eq)] pub struct Output { + // pub_key of the derived_key that used for this output pub unique_pubkey: UniquePubkey, pub amount: NanoTokens, + // Purpose of this output, i.e. the reason why the output got generated. + pub purpose: String, } impl Output { - pub fn new(unique_pubkey: UniquePubkey, amount: u64) -> Self { + pub fn new(unique_pubkey: UniquePubkey, amount: u64, purpose: String) -> Self { Self { unique_pubkey, amount: NanoTokens::from(amount), + purpose, } } @@ -57,6 +62,7 @@ impl Output { let mut v: Vec = Default::default(); v.extend(self.unique_pubkey.to_bytes().as_ref()); v.extend(self.amount.to_bytes()); + v.extend(self.purpose.clone().into_bytes()); v } From a3a2044168dcb7e527f116670d883780bcd2d4be Mon Sep 17 00:00:00 2001 From: qima Date: Tue, 30 Apr 2024 20:31:45 +0800 Subject: [PATCH 172/205] chore: store owner info inside node instead of network --- sn_client/src/api.rs | 14 ++------------ sn_client/src/uploader/tests/setup.rs | 7 +------ sn_networking/src/driver.rs | 11 +---------- sn_networking/src/lib.rs | 17 +++-------------- sn_node/src/node.rs | 11 ++++++++--- sn_node/src/quote.rs | 5 +++-- 6 files changed, 18 insertions(+), 47 deletions(-) diff --git a/sn_client/src/api.rs b/sn_client/src/api.rs index 44afb567df..2e171ebf7f 100644 --- a/sn_client/src/api.rs +++ b/sn_client/src/api.rs @@ -108,20 +108,10 @@ impl Client { // TODO: shall client bearing owner's discord user name, to be reflected in the cash_notes? #[cfg(not(feature = "open-metrics"))] - let network_builder = NetworkBuilder::new( - Keypair::generate_ed25519(), - local, - root_dir, - "maidsafe_client".to_string(), - ); + let network_builder = NetworkBuilder::new(Keypair::generate_ed25519(), local, root_dir); #[cfg(feature = "open-metrics")] - let mut network_builder = NetworkBuilder::new( - Keypair::generate_ed25519(), - local, - root_dir, - "maidsafe_client".to_string(), - ); + let mut network_builder = NetworkBuilder::new(Keypair::generate_ed25519(), local, root_dir); #[cfg(feature = "open-metrics")] network_builder.metrics_registry(Registry::default()); diff --git a/sn_client/src/uploader/tests/setup.rs b/sn_client/src/uploader/tests/setup.rs index 3e8d678d7c..15dc536685 100644 --- a/sn_client/src/uploader/tests/setup.rs +++ b/sn_client/src/uploader/tests/setup.rs @@ -425,12 +425,7 @@ pub fn start_uploading_with_steps( // Build a very simple client struct for testing. This does not connect to any network. // The UploaderInterface eliminates the need for direct networking in tests. pub fn build_unconnected_client(root_dir: PathBuf) -> Result { - let network_builder = NetworkBuilder::new( - Keypair::generate_ed25519(), - true, - root_dir, - "maidsafe_client".to_string(), - ); + let network_builder = NetworkBuilder::new(Keypair::generate_ed25519(), true, root_dir); let (network, ..) = network_builder.build_client()?; let client = Client { network: network.clone(), diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs index 1d1af4b7fc..c104ec14cb 100644 --- a/sn_networking/src/driver.rs +++ b/sn_networking/src/driver.rs @@ -206,11 +206,10 @@ pub struct NetworkBuilder { metrics_registry: Option, #[cfg(feature = "open-metrics")] metrics_server_port: u16, - owner: String, } impl NetworkBuilder { - pub fn new(keypair: Keypair, local: bool, root_dir: PathBuf, owner: String) -> Self { + pub fn new(keypair: Keypair, local: bool, root_dir: PathBuf) -> Self { Self { is_behind_home_network: false, keypair, @@ -224,7 +223,6 @@ impl NetworkBuilder { metrics_registry: None, #[cfg(feature = "open-metrics")] metrics_server_port: 0, - owner, } } @@ -317,7 +315,6 @@ impl NetworkBuilder { }; let listen_addr = self.listen_addr; - let owner = self.owner.clone(); let (network, events_receiver, mut swarm_driver) = self.build( kad_cfg, @@ -325,7 +322,6 @@ impl NetworkBuilder { false, ProtocolSupport::Full, IDENTIFY_NODE_VERSION_STR.to_string(), - owner, )?; // Listen on the provided address @@ -371,15 +367,12 @@ impl NetworkBuilder { .ok_or_else(|| NetworkError::InvalidCloseGroupSize)?, ); - let owner = self.owner.clone(); - let (network, net_event_recv, driver) = self.build( kad_cfg, None, true, ProtocolSupport::Outbound, IDENTIFY_CLIENT_VERSION_STR.to_string(), - owner, )?; Ok((network, net_event_recv, driver)) @@ -393,7 +386,6 @@ impl NetworkBuilder { is_client: bool, req_res_protocol: ProtocolSupport, identify_version: String, - owner: String, ) -> Result<(Network, mpsc::Receiver, SwarmDriver)> { let peer_id = PeerId::from(self.keypair.public()); // vdash metric (if modified please notify at https://github.com/happybeing/vdash/issues): @@ -587,7 +579,6 @@ impl NetworkBuilder { peer_id: Arc::new(peer_id), root_dir_path: Arc::new(self.root_dir), keypair: Arc::new(self.keypair), - owner, }, network_event_receiver, swarm_driver, diff --git a/sn_networking/src/lib.rs b/sn_networking/src/lib.rs index 23943e91c2..75d51ab1e2 100644 --- a/sn_networking/src/lib.rs +++ b/sn_networking/src/lib.rs @@ -158,16 +158,9 @@ pub struct Network { pub peer_id: Arc, pub root_dir_path: Arc, keypair: Arc, - /// node owner's discord username, in readable format - owner: String, } impl Network { - /// Returns of the discord user_name of the node's owner - pub fn owner(&self) -> String { - self.owner.clone() - } - /// Signs the given data with the node's keypair. pub fn sign(&self, msg: &[u8]) -> Result> { self.keypair.sign(msg).map_err(NetworkError::from) @@ -1053,13 +1046,9 @@ mod tests { #[test] fn test_network_sign_verify() -> eyre::Result<()> { - let (network, _, _) = NetworkBuilder::new( - Keypair::generate_ed25519(), - false, - std::env::temp_dir(), - "maidsafe_test".to_string(), - ) - .build_client()?; + let (network, _, _) = + NetworkBuilder::new(Keypair::generate_ed25519(), false, std::env::temp_dir()) + .build_client()?; let msg = b"test message"; let sig = network.sign(msg)?; assert!(network.verify(msg, &sig)); diff --git a/sn_node/src/node.rs b/sn_node/src/node.rs index ace0473d59..559e41d081 100644 --- a/sn_node/src/node.rs +++ b/sn_node/src/node.rs @@ -133,8 +133,7 @@ impl NodeBuilder { (metrics_registry, node_metrics) }; - let mut network_builder = - NetworkBuilder::new(self.keypair, self.local, self.root_dir, self.owner.clone()); + let mut network_builder = NetworkBuilder::new(self.keypair, self.local, self.root_dir); network_builder.listen_addr(self.addr); #[cfg(feature = "open-metrics")] @@ -156,6 +155,7 @@ impl NodeBuilder { reward_address: Arc::new(reward_address), #[cfg(feature = "open-metrics")] node_metrics, + owner: self.owner.clone(), }; let running_node = RunningNode { network, @@ -188,6 +188,8 @@ pub(crate) struct Node { reward_address: Arc, #[cfg(feature = "open-metrics")] pub(crate) node_metrics: NodeMetrics, + /// node owner's discord username, in readable format + owner: String, } impl Node { @@ -389,9 +391,10 @@ impl Node { event_header = "QueryRequestReceived"; let network = self.network.clone(); let payment_address = *self.reward_address; + let owner = self.owner.clone(); let _handle = spawn(async move { - let res = Self::handle_query(&network, query, payment_address).await; + let res = Self::handle_query(&network, query, payment_address, owner).await; trace!("Sending response {res:?}"); network.send_response(res, channel); @@ -564,6 +567,7 @@ impl Node { network: &Network, query: Query, payment_address: MainPubkey, + owner: String, ) -> Response { let resp: QueryResponse = match query { Query::GetStoreCost(address) => { @@ -590,6 +594,7 @@ impl Node { cost, &address, "ing_metrics, + owner, ), payment_address, peer_address: NetworkAddress::from_peer(self_id), diff --git a/sn_node/src/quote.rs b/sn_node/src/quote.rs index a673e0afba..5f19f95cdc 100644 --- a/sn_node/src/quote.rs +++ b/sn_node/src/quote.rs @@ -19,6 +19,7 @@ impl Node { cost: NanoTokens, address: &NetworkAddress, quoting_metrics: &QuotingMetrics, + owner: String, ) -> Result { let content = address.as_xorname().unwrap_or_default(); let timestamp = std::time::SystemTime::now(); @@ -27,7 +28,7 @@ impl Node { cost, timestamp, quoting_metrics, - network.owner(), + owner.clone(), ); let Ok(signature) = network.sign(&bytes) else { @@ -39,7 +40,7 @@ impl Node { cost, timestamp, quoting_metrics: quoting_metrics.clone(), - reason: network.owner(), + reason: owner, pub_key: network.get_pub_key(), signature, }; From 5252242d0cbdaf9cbc1a7c58983128a7921a2d36 Mon Sep 17 00:00:00 2001 From: qima Date: Tue, 30 Apr 2024 22:10:52 +0800 Subject: [PATCH 173/205] chore: rename output reason to purpose for clarity --- sn_networking/src/transfers.rs | 6 +++--- sn_node/src/quote.rs | 4 ++-- sn_transfers/src/cashnotes.rs | 8 ++++---- sn_transfers/src/cashnotes/builder.rs | 16 ++++++++-------- sn_transfers/src/cashnotes/cashnote.rs | 14 +++++++------- sn_transfers/src/transfers/offline_transfer.rs | 8 ++++---- sn_transfers/src/transfers/transfer.rs | 8 ++++---- sn_transfers/src/wallet/data_payments.rs | 16 ++++++++-------- sn_transfers/src/wallet/hot_wallet.rs | 18 +++++++++--------- sn_transfers/src/wallet/watch_only.rs | 4 ++-- 10 files changed, 51 insertions(+), 51 deletions(-) diff --git a/sn_networking/src/transfers.rs b/sn_networking/src/transfers.rs index 5886281892..ece1b99cd7 100644 --- a/sn_networking/src/transfers.rs +++ b/sn_networking/src/transfers.rs @@ -148,12 +148,12 @@ impl Network { .iter() .map(|u| { let unique_pubkey = main_pubkey.new_unique_pubkey(&u.derivation_index); - (u.reason.clone(), unique_pubkey, u.derivation_index) + (u.purpose.clone(), unique_pubkey, u.derivation_index) }) .collect(); let mut our_output_cash_notes = Vec::new(); - for (reason, id, derivation_index) in our_output_unique_pubkeys.into_iter() { + for (purpose, id, derivation_index) in our_output_unique_pubkeys.into_iter() { let src_tx = parent_txs .iter() .find(|tx| tx.outputs.iter().any(|o| o.unique_pubkey() == &id)) @@ -170,7 +170,7 @@ impl Network { unique_pubkey: id, parent_tx: src_tx, parent_spends: signed_spends, - reason: reason.clone(), + purpose: purpose.clone(), main_pubkey, derivation_index, }; diff --git a/sn_node/src/quote.rs b/sn_node/src/quote.rs index 5f19f95cdc..3036b7c94c 100644 --- a/sn_node/src/quote.rs +++ b/sn_node/src/quote.rs @@ -40,7 +40,7 @@ impl Node { cost, timestamp, quoting_metrics: quoting_metrics.clone(), - reason: owner, + owner, pub_key: network.get_pub_key(), signature, }; @@ -73,7 +73,7 @@ pub(crate) fn verify_quote_for_storecost( quote.cost, quote.timestamp, "e.quoting_metrics, - quote.reason.clone(), + quote.owner.clone(), ); let signature = quote.signature; if !network.verify(&bytes, &signature) { diff --git a/sn_transfers/src/cashnotes.rs b/sn_transfers/src/cashnotes.rs index 428e329d23..baacd00590 100644 --- a/sn_transfers/src/cashnotes.rs +++ b/sn_transfers/src/cashnotes.rs @@ -53,7 +53,7 @@ pub(crate) mod tests { unique_pubkey: derived_key.unique_pubkey(), parent_tx: tx, parent_spends: Default::default(), - reason: Default::default(), + purpose: Default::default(), main_pubkey: main_key.main_pubkey(), derivation_index, }; @@ -85,7 +85,7 @@ pub(crate) mod tests { unique_pubkey: derived_key.unique_pubkey(), parent_tx: tx, parent_spends: Default::default(), - reason: Default::default(), + purpose: Default::default(), main_pubkey: main_key.main_pubkey(), derivation_index, }; @@ -121,7 +121,7 @@ pub(crate) mod tests { unique_pubkey: derived_key.unique_pubkey(), parent_tx: tx, parent_spends: Default::default(), - reason: Default::default(), + purpose: Default::default(), main_pubkey: main_key.main_pubkey(), derivation_index, }; @@ -157,7 +157,7 @@ pub(crate) mod tests { unique_pubkey: derived_key.unique_pubkey(), parent_tx: tx, parent_spends: Default::default(), - reason: Default::default(), + purpose: Default::default(), main_pubkey: main_key.main_pubkey(), derivation_index, }; diff --git a/sn_transfers/src/cashnotes/builder.rs b/sn_transfers/src/cashnotes/builder.rs index 471190c8ab..799fbbc396 100644 --- a/sn_transfers/src/cashnotes/builder.rs +++ b/sn_transfers/src/cashnotes/builder.rs @@ -72,11 +72,11 @@ impl TransactionBuilder { self } - /// Add an output given the token, reason, the MainPubkey and the DerivationIndex + /// Add an output given the token, purpose, the MainPubkey and the DerivationIndex pub fn add_output( mut self, token: NanoTokens, - reason: String, + purpose: String, main_pubkey: MainPubkey, derivation_index: DerivationIndex, ) -> Self { @@ -84,9 +84,9 @@ impl TransactionBuilder { self.output_details.insert( unique_pubkey, - (reason.clone(), main_pubkey, derivation_index), + (purpose.clone(), main_pubkey, derivation_index), ); - let output = Output::new(unique_pubkey, token.as_nano(), reason); + let output = Output::new(unique_pubkey, token.as_nano(), purpose); self.outputs.push(output); self @@ -97,8 +97,8 @@ impl TransactionBuilder { mut self, outputs: impl IntoIterator, ) -> Self { - for (token, reason, main_pubkey, derivation_index) in outputs.into_iter() { - self = self.add_output(token, reason, main_pubkey, derivation_index); + for (token, purpose, main_pubkey, derivation_index) in outputs.into_iter() { + self = self.add_output(token, purpose, main_pubkey, derivation_index); } self } @@ -231,7 +231,7 @@ impl CashNoteBuilder { .outputs .iter() .map(|output| { - let (reason, main_pubkey, derivation_index) = self + let (purpose, main_pubkey, derivation_index) = self .output_details .get(&output.unique_pubkey) .ok_or(TransferError::UniquePubkeyNotFound)?; @@ -241,7 +241,7 @@ impl CashNoteBuilder { unique_pubkey: main_pubkey.new_unique_pubkey(derivation_index), parent_tx: self.spent_tx.clone(), parent_spends: self.signed_spends.clone(), - reason: reason.clone(), + purpose: purpose.clone(), main_pubkey: *main_pubkey, derivation_index: *derivation_index, }, diff --git a/sn_transfers/src/cashnotes/cashnote.rs b/sn_transfers/src/cashnotes/cashnote.rs index 84fcd00dda..5d992fbaf1 100644 --- a/sn_transfers/src/cashnotes/cashnote.rs +++ b/sn_transfers/src/cashnotes/cashnote.rs @@ -69,9 +69,9 @@ pub struct CashNote { pub parent_tx: Transaction, /// The transaction's input's SignedSpends pub parent_spends: BTreeSet, - /// The reason this cash_note created for + /// The purpose this cash_note created for /// eg. `store cost pay to...`, `network royalty`, `change to self`, `payment transfer`, ... - pub reason: String, + pub purpose: String, /// This is the MainPubkey of the owner of this CashNote pub main_pubkey: MainPubkey, /// The derivation index used to derive the UniquePubkey and DerivedSecretKey from the MainPubkey and MainSecretKey respectively. @@ -122,13 +122,13 @@ impl CashNote { self.parent_spends .iter() .next() - .map(|c| c.reason()) + .map(|s| s.reason()) .unwrap_or_default() } - /// Return the reason why this CashNote was created. - pub fn self_reason(&self) -> String { - self.reason.clone() + /// Return the purpose why this CashNote was created. + pub fn purpose(&self) -> String { + self.purpose.clone() } /// Return the value in NanoTokens for this CashNote. @@ -153,7 +153,7 @@ impl CashNote { sha3.update(&sp.to_bytes()); } - sha3.update(&self.reason.clone().into_bytes()); + sha3.update(&self.purpose.clone().into_bytes()); sha3.update(self.spent_reason().as_ref()); let mut hash = [0u8; 32]; diff --git a/sn_transfers/src/transfers/offline_transfer.rs b/sn_transfers/src/transfers/offline_transfer.rs index bd2c6b32f6..a81d9130ab 100644 --- a/sn_transfers/src/transfers/offline_transfer.rs +++ b/sn_transfers/src/transfers/offline_transfer.rs @@ -93,7 +93,7 @@ impl OfflineTransfer { available_cash_notes: CashNotesAndSecretKey, recipients: Vec<(NanoTokens, String, MainPubkey, DerivationIndex)>, change_to: MainPubkey, - reason_hash: Hash, + input_reason_hash: Hash, ) -> Result { let total_output_amount = recipients .iter() @@ -116,7 +116,7 @@ impl OfflineTransfer { change: (change_amount, change_to), }; - create_offline_transfer_with(selected_inputs, reason_hash) + create_offline_transfer_with(selected_inputs, input_reason_hash) } } @@ -294,7 +294,7 @@ fn create_transaction_builder_with( /// enough peers in the network, the transaction will be completed. fn create_offline_transfer_with( selected_inputs: TransferInputs, - reason_hash: Hash, + input_reason_hash: Hash, ) -> Result { // gather the network_royalties derivation indexes let network_royalties: Vec = selected_inputs @@ -307,7 +307,7 @@ fn create_offline_transfer_with( let (tx_builder, src_txs, change_id) = create_transaction_builder_with(selected_inputs)?; // Finalize the tx builder to get the cash_note builder. - let cash_note_builder = tx_builder.build(reason_hash, network_royalties)?; + let cash_note_builder = tx_builder.build(input_reason_hash, network_royalties)?; let tx = cash_note_builder.spent_tx.clone(); diff --git a/sn_transfers/src/transfers/transfer.rs b/sn_transfers/src/transfers/transfer.rs index 514ed716de..37928f1d56 100644 --- a/sn_transfers/src/transfers/transfer.rs +++ b/sn_transfers/src/transfers/transfer.rs @@ -139,7 +139,7 @@ pub struct CashNoteRedemption { /// using data found at this address the owner can check that the output is valid money pub parent_spend: SpendAddress, /// For what purpose this cash_note was created - pub reason: String, + pub purpose: String, } impl CashNoteRedemption { @@ -147,12 +147,12 @@ impl CashNoteRedemption { pub fn new( derivation_index: DerivationIndex, parent_spend: SpendAddress, - reason: String, + purpose: String, ) -> Self { Self { derivation_index, parent_spend, - reason, + purpose, } } @@ -167,7 +167,7 @@ impl CashNoteRedemption { Ok(Self::new( derivation_index, parent_spend, - cash_note.reason.clone(), + cash_note.purpose.clone(), )) } diff --git a/sn_transfers/src/wallet/data_payments.rs b/sn_transfers/src/wallet/data_payments.rs index b7e0164522..dc617e1ff6 100644 --- a/sn_transfers/src/wallet/data_payments.rs +++ b/sn_transfers/src/wallet/data_payments.rs @@ -106,8 +106,8 @@ pub struct PaymentQuote { pub timestamp: SystemTime, /// quoting metrics being used to generate this quote pub quoting_metrics: QuotingMetrics, - /// node's reason to accept the payment. Normally using its discord username - pub reason: String, + /// node's owner to accept the payment. Normally using its discord username + pub owner: String, /// node's public key that can verify the signature #[debug(skip)] pub pub_key: Vec, @@ -123,7 +123,7 @@ impl PaymentQuote { cost: NanoTokens::zero(), timestamp: SystemTime::now(), quoting_metrics: Default::default(), - reason: Default::default(), + owner: Default::default(), pub_key: vec![], signature: vec![], } @@ -135,7 +135,7 @@ impl PaymentQuote { cost: NanoTokens, timestamp: SystemTime, quoting_metrics: &QuotingMetrics, - reason: String, + owner: String, ) -> Vec { let mut bytes = xorname.to_vec(); bytes.extend_from_slice(&cost.to_bytes()); @@ -151,7 +151,7 @@ impl PaymentQuote { Err(_err) => vec![], }; bytes.extend_from_slice(&serialised_quoting_metrics); - bytes.extend_from_slice(&reason.into_bytes()); + bytes.extend_from_slice(&owner.into_bytes()); bytes } @@ -176,7 +176,7 @@ impl PaymentQuote { self.cost, self.timestamp, &self.quoting_metrics, - self.reason.clone(), + self.owner.clone(), ); if !pub_key.verify(&bytes, &self.signature) { @@ -205,7 +205,7 @@ impl PaymentQuote { cost, timestamp: SystemTime::now(), quoting_metrics: Default::default(), - reason: Default::default(), + owner: Default::default(), pub_key: vec![], signature: vec![], } @@ -304,7 +304,7 @@ mod tests { quote.cost, quote.timestamp, "e.quoting_metrics, - quote.reason.clone(), + quote.owner.clone(), ); let signature = if let Ok(sig) = keypair.sign(&bytes) { sig diff --git a/sn_transfers/src/wallet/hot_wallet.rs b/sn_transfers/src/wallet/hot_wallet.rs index 563d601fbb..55a0ff4999 100644 --- a/sn_transfers/src/wallet/hot_wallet.rs +++ b/sn_transfers/src/wallet/hot_wallet.rs @@ -37,7 +37,7 @@ use xor_name::XorName; /// A locked file handle, that when dropped releases the lock. pub type WalletExclusiveAccess = File; -// TransactionPayeeDetails: (reason, amount, address) +// TransactionPayeeDetails: (purpose, amount, address) pub type TransactionPayeeDetails = (String, NanoTokens, MainPubkey); /// A hot-wallet. @@ -323,8 +323,8 @@ impl HotWallet { // create a unique key for each output let to_unique_keys: Vec<_> = to .into_iter() - .map(|(reason, amount, address)| { - (amount, reason, address, DerivationIndex::random(&mut rng)) + .map(|(purpose, amount, address)| { + (amount, purpose, address, DerivationIndex::random(&mut rng)) }) .collect(); @@ -394,7 +394,7 @@ impl HotWallet { for (xorname, (main_pubkey, quote, peer_id_bytes)) in price_map.iter() { let storage_payee = ( quote.cost, - quote.reason.clone(), + quote.owner.clone(), *main_pubkey, DerivationIndex::random(&mut rng), peer_id_bytes.clone(), @@ -421,9 +421,9 @@ impl HotWallet { let recipients = recipients_by_xor .values() .flat_map( - |((cost, reason, pubkey, derivation_index, _id_bytes), roy)| { + |((cost, purpose, pubkey, derivation_index, _id_bytes), roy)| { vec![ - (*cost, reason.clone(), *pubkey, *derivation_index), + (*cost, purpose.clone(), *pubkey, *derivation_index), roy.clone(), ] }, @@ -443,13 +443,13 @@ impl HotWallet { start.elapsed() ); debug!("Available CashNotes: {:#?}", available_cash_notes); - let reason_hash = Hash::hash(b"SPEND_REASON_FOR_STORAGE"); + let input_reason_hash = Hash::hash(b"SPEND_REASON_FOR_STORAGE"); let start = Instant::now(); let offline_transfer = OfflineTransfer::new( available_cash_notes, recipients, self.address(), - reason_hash, + input_reason_hash, )?; trace!( "local_send_storage_payment created offline_transfer with {} cashnotes in {:?}", @@ -466,7 +466,7 @@ impl HotWallet { .collect(); for (xorname, recipients_info) in recipients_by_xor { let (storage_payee, royalties_payee) = recipients_info; - let (pay_amount, _reason, node_key, _, peer_id_bytes) = storage_payee; + let (pay_amount, _purpose, node_key, _, peer_id_bytes) = storage_payee; let cash_note_for_node = cashnotes_to_use .iter() .find(|cash_note| { diff --git a/sn_transfers/src/wallet/watch_only.rs b/sn_transfers/src/wallet/watch_only.rs index bea31b9aaf..34c615eab3 100644 --- a/sn_transfers/src/wallet/watch_only.rs +++ b/sn_transfers/src/wallet/watch_only.rs @@ -222,8 +222,8 @@ impl WatchOnlyWallet { // create a unique key for each output let to_unique_keys: Vec<_> = to .into_iter() - .map(|(reason, amount, address)| { - (amount, reason, address, DerivationIndex::random(&mut rng)) + .map(|(purpose, amount, address)| { + (amount, purpose, address, DerivationIndex::random(&mut rng)) }) .collect(); From c39da28bd87a87ed610b8d98bf468c566c87f851 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Wed, 17 Apr 2024 11:23:24 +0900 Subject: [PATCH 174/205] feat: init of nodeui --- Cargo.lock | 1096 ++++++++++++++++++++++++++++- Cargo.toml | 1 + node-tui/.config/config.json5 | 10 + node-tui/.envrc | 3 + node-tui/.github/workflows/cd.yml | 152 ++++ node-tui/.github/workflows/ci.yml | 63 ++ node-tui/.gitignore | 2 + node-tui/.rustfmt.toml | 16 + node-tui/Cargo.toml | 48 ++ node-tui/LICENSE | 21 + node-tui/README.md | 5 + node-tui/build.rs | 4 + node-tui/rust-toolchain.toml | 2 + node-tui/src/action.rs | 21 + node-tui/src/app.rs | 151 ++++ node-tui/src/cli.rs | 27 + node-tui/src/components.rs | 124 ++++ node-tui/src/components/fps.rs | 90 +++ node-tui/src/components/home.rs | 100 +++ node-tui/src/config.rs | 504 +++++++++++++ node-tui/src/main.rs | 41 ++ node-tui/src/mode.rs | 7 + node-tui/src/tui.rs | 239 +++++++ node-tui/src/utils.rs | 161 +++++ 24 files changed, 2871 insertions(+), 17 deletions(-) create mode 100644 node-tui/.config/config.json5 create mode 100644 node-tui/.envrc create mode 100644 node-tui/.github/workflows/cd.yml create mode 100644 node-tui/.github/workflows/ci.yml create mode 100644 node-tui/.gitignore create mode 100644 node-tui/.rustfmt.toml create mode 100644 node-tui/Cargo.toml create mode 100644 node-tui/LICENSE create mode 100644 node-tui/README.md create mode 100644 node-tui/build.rs create mode 100644 node-tui/rust-toolchain.toml create mode 100644 node-tui/src/action.rs create mode 100644 node-tui/src/app.rs create mode 100644 node-tui/src/cli.rs create mode 100644 node-tui/src/components.rs create mode 100644 node-tui/src/components/fps.rs create mode 100644 node-tui/src/components/home.rs create mode 100644 node-tui/src/config.rs create mode 100644 node-tui/src/main.rs create mode 100644 node-tui/src/mode.rs create mode 100644 node-tui/src/tui.rs create mode 100644 node-tui/src/utils.rs diff --git a/Cargo.lock b/Cargo.lock index ff6159cf44..4ac8ec3783 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -197,6 +197,12 @@ version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "arrayref" version = "0.3.7" @@ -521,6 +527,16 @@ version = "0.10.0-beta" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98f7eed2b2781a6f0b5c903471d48e15f56fb4e1165df8a9a2337fd1a59d45ea" +[[package]] +name = "better-panic" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fa9e1d11a268684cbd90ed36370d7577afb6c62d912ddff5c15fc34343e5036" +dependencies = [ + "backtrace", + "console", +] + [[package]] name = "bincode" version = "1.3.3" @@ -604,6 +620,9 @@ name = "bitflags" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +dependencies = [ + "serde", +] [[package]] name = "bitvec" @@ -781,6 +800,15 @@ dependencies = [ "serde", ] +[[package]] +name = "btoi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd6407f73a9b8b6162d8a2ef999fe6afd7cc15902ebf42c5cd296addf17e0ad" +dependencies = [ + "num-traits", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -841,12 +869,59 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "camino" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + [[package]] name = "cast" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "castaway" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a17ed5635fc8536268e5d4de1e22e81ac34419e5f052d4d51f4e01dcc263fcc" +dependencies = [ + "rustversion", +] + [[package]] name = "cbc" version = "0.1.2" @@ -993,6 +1068,9 @@ dependencies = [ "anstyle", "clap_lex", "strsim 0.11.1", + "terminal_size", + "unicase", + "unicode-width", ] [[package]] @@ -1022,6 +1100,12 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "clru" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8191fa7302e03607ff0e237d4246cc043ff5b3cb9409d995172ba3bea16b807" + [[package]] name = "color-eyre" version = "0.6.3" @@ -1065,6 +1149,20 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "compact_str" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f86b9c4c00838774a6d902ef931eff7470720c51d90c2e32cfe15dc304737b3f" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "ryu", + "serde", + "static_assertions", +] + [[package]] name = "concurrent-queue" version = "2.4.0" @@ -1074,6 +1172,26 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "config" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7328b20597b53c2454f0b1919720c25c7339051c02b72b7e05409e00b14132be" +dependencies = [ + "async-trait", + "convert_case", + "json5", + "lazy_static", + "nom", + "pathdiff", + "ron", + "rust-ini", + "serde", + "serde_json", + "toml", + "yaml-rust", +] + [[package]] name = "console" version = "0.15.8" @@ -1103,12 +1221,41 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.14", + "once_cell", + "tiny-keccak", +] + [[package]] name = "constant_time_eq" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1243,6 +1390,33 @@ version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +[[package]] +name = "crossterm" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" +dependencies = [ + "bitflags 2.5.0", + "crossterm_winapi", + "futures-core", + "libc", + "mio", + "parking_lot", + "serde", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + [[package]] name = "crunchy" version = "0.2.2" @@ -1468,6 +1642,17 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_deref" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcdbcee2d9941369faba772587a565f4f534e42cb8d17e5295871de730163b2b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "dialoguer" version = "0.11.0" @@ -1481,6 +1666,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "difflib" version = "0.4.0" @@ -1516,13 +1707,22 @@ dependencies = [ "subtle", ] +[[package]] +name = "directories" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" +dependencies = [ + "dirs-sys 0.4.1", +] + [[package]] name = "dirs" version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" dependencies = [ - "dirs-sys", + "dirs-sys 0.3.7", ] [[package]] @@ -1546,6 +1746,18 @@ dependencies = [ "winapi", ] +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -1568,6 +1780,15 @@ dependencies = [ "syn 2.0.60", ] +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + [[package]] name = "doc-comment" version = "0.3.3" @@ -1601,6 +1822,12 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" +[[package]] +name = "dunce" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" + [[package]] name = "ecdsa" version = "0.14.8" @@ -1768,6 +1995,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +[[package]] +name = "faster-hex" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2a2b11eda1d40935b26cf18f6833c526845ae8c41e58d09af6adeb6f0269183" + [[package]] name = "fastrand" version = "2.0.2" @@ -2091,33 +2324,533 @@ dependencies = [ ] [[package]] -name = "getrandom" -version = "0.2.14" +name = "getrandom" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug 0.3.1", + "polyval", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "gix" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd025382892c7b500a9ce1582cd803f9c2ebfe44aff52e9c7f86feee7ced75e" +dependencies = [ + "gix-actor", + "gix-commitgraph", + "gix-config", + "gix-date", + "gix-diff", + "gix-discover", + "gix-features", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-hashtable", + "gix-index", + "gix-lock", + "gix-macros", + "gix-object", + "gix-odb", + "gix-pack", + "gix-path", + "gix-ref", + "gix-refspec", + "gix-revision", + "gix-revwalk", + "gix-sec", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-url", + "gix-utils", + "gix-validate", + "once_cell", + "parking_lot", + "signal-hook", + "smallvec", + "thiserror", + "unicode-normalization", +] + +[[package]] +name = "gix-actor" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da27b5ab4ab5c75ff891dccd48409f8cc53c28a79480f1efdd33184b2dc1d958" +dependencies = [ + "bstr", + "btoi", + "gix-date", + "itoa", + "thiserror", + "winnow 0.5.40", +] + +[[package]] +name = "gix-bitmap" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a371db66cbd4e13f0ed9dc4c0fea712d7276805fccc877f77e96374d317e87ae" +dependencies = [ + "thiserror", +] + +[[package]] +name = "gix-chunk" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45c8751169961ba7640b513c3b24af61aa962c967aaf04116734975cd5af0c52" +dependencies = [ + "thiserror", +] + +[[package]] +name = "gix-commitgraph" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8dcbf434951fa477063e05fea59722615af70dc2567377e58c2f7853b010fc" +dependencies = [ + "bstr", + "gix-chunk", + "gix-features", + "gix-hash", + "memmap2", + "thiserror", +] + +[[package]] +name = "gix-config" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "367304855b369cadcac4ee5fb5a3a20da9378dd7905106141070b79f85241079" +dependencies = [ + "bstr", + "gix-config-value", + "gix-features", + "gix-glob", + "gix-path", + "gix-ref", + "gix-sec", + "memchr", + "once_cell", + "smallvec", + "thiserror", + "unicode-bom", + "winnow 0.5.40", +] + +[[package]] +name = "gix-config-value" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbd06203b1a9b33a78c88252a625031b094d9e1b647260070c25b09910c0a804" +dependencies = [ + "bitflags 2.5.0", + "bstr", + "gix-path", + "libc", + "thiserror", +] + +[[package]] +name = "gix-date" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180b130a4a41870edfbd36ce4169c7090bca70e195da783dea088dd973daa59c" +dependencies = [ + "bstr", + "itoa", + "thiserror", + "time", +] + +[[package]] +name = "gix-diff" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6a0454f8c42d686f17e7f084057c717c082b7dbb8209729e4e8f26749eb93a" +dependencies = [ + "bstr", + "gix-hash", + "gix-object", + "thiserror", +] + +[[package]] +name = "gix-discover" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8d7b2896edc3d899d28a646ccc6df729827a6600e546570b2783466404a42d6" +dependencies = [ + "bstr", + "dunce", + "gix-hash", + "gix-path", + "gix-ref", + "gix-sec", + "thiserror", +] + +[[package]] +name = "gix-features" +version = "0.37.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d50270e8dcc665f30ba0735b17984b9535bdf1e646c76e638e007846164d57af" +dependencies = [ + "crc32fast", + "flate2", + "gix-hash", + "gix-trace", + "libc", + "once_cell", + "prodash", + "sha1_smol", + "thiserror", + "walkdir", +] + +[[package]] +name = "gix-fs" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7555c23a005537434bbfcb8939694e18cad42602961d0de617f8477cc2adecdd" +dependencies = [ + "gix-features", +] + +[[package]] +name = "gix-glob" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae6232f18b262770e343dcdd461c0011c9b9ae27f0c805e115012aa2b902c1b8" +dependencies = [ + "bitflags 2.5.0", + "bstr", + "gix-features", + "gix-path", +] + +[[package]] +name = "gix-hash" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93d7df7366121b5018f947a04d37f034717e113dcf9ccd85c34b58e57a74d5e" +dependencies = [ + "faster-hex", + "thiserror", +] + +[[package]] +name = "gix-hashtable" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ddf80e16f3c19ac06ce415a38b8591993d3f73aede049cb561becb5b3a8e242" +dependencies = [ + "gix-hash", + "hashbrown 0.14.3", + "parking_lot", +] + +[[package]] +name = "gix-index" +version = "0.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e50e63df6c8d4137f7fb882f27643b3a9756c468a1a2cdbe1ce443010ca8778" +dependencies = [ + "bitflags 2.5.0", + "bstr", + "btoi", + "filetime", + "gix-bitmap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-traverse", + "itoa", + "libc", + "memmap2", + "rustix", + "smallvec", + "thiserror", +] + +[[package]] +name = "gix-lock" +version = "12.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40a439397f1e230b54cf85d52af87e5ea44cc1e7748379785d3f6d03d802b00" +dependencies = [ + "gix-tempfile", + "gix-utils", + "thiserror", +] + +[[package]] +name = "gix-macros" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dff438f14e67e7713ab9332f5fd18c8f20eb7eb249494f6c2bf170522224032" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "gix-object" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c89402e8faa41b49fde348665a8f38589e461036475af43b6b70615a6a313a2" +dependencies = [ + "bstr", + "btoi", + "gix-actor", + "gix-date", + "gix-features", + "gix-hash", + "gix-validate", + "itoa", + "smallvec", + "thiserror", + "winnow 0.5.40", +] + +[[package]] +name = "gix-odb" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46ae6da873de41c6c2b73570e82c571b69df5154dcd8f46dfafc6687767c33b1" +dependencies = [ + "arc-swap", + "gix-date", + "gix-features", + "gix-hash", + "gix-object", + "gix-pack", + "gix-path", + "gix-quote", + "parking_lot", + "tempfile", + "thiserror", +] + +[[package]] +name = "gix-pack" +version = "0.46.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "782b4d42790a14072d5c400deda9851f5765f50fe72bca6dece0da1cd6f05a9a" +dependencies = [ + "clru", + "gix-chunk", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-path", + "gix-tempfile", + "memmap2", + "parking_lot", + "smallvec", + "thiserror", +] + +[[package]] +name = "gix-path" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23623cf0f475691a6d943f898c4d0b89f5c1a2a64d0f92bce0e0322ee6528783" +dependencies = [ + "bstr", + "gix-trace", + "home", + "once_cell", + "thiserror", +] + +[[package]] +name = "gix-quote" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbff4f9b9ea3fa7a25a70ee62f545143abef624ac6aa5884344e70c8b0a1d9ff" +dependencies = [ + "bstr", + "gix-utils", + "thiserror", +] + +[[package]] +name = "gix-ref" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d9bd1984638d8f3511a2fcbe84fcedb8a5b5d64df677353620572383f42649" +dependencies = [ + "gix-actor", + "gix-date", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-validate", + "memmap2", + "thiserror", + "winnow 0.5.40", +] + +[[package]] +name = "gix-refspec" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be219df5092c1735abb2a53eccdf775e945eea6986ee1b6e7a5896dccc0be704" +dependencies = [ + "bstr", + "gix-hash", + "gix-revision", + "gix-validate", + "smallvec", + "thiserror", +] + +[[package]] +name = "gix-revision" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa78e1df3633bc937d4db15f8dca2abdb1300ca971c0fabcf9fa97e38cf4cd9f" +dependencies = [ + "bstr", + "gix-date", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "gix-trace", + "thiserror", +] + +[[package]] +name = "gix-revwalk" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "702de5fe5c2bbdde80219f3a8b9723eb927466e7ecd187cfd1b45d986408e45f" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-hashtable", + "gix-object", + "smallvec", + "thiserror", +] + +[[package]] +name = "gix-sec" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fddc27984a643b20dd03e97790555804f98cf07404e0e552c0ad8133266a79a1" +dependencies = [ + "bitflags 2.5.0", + "gix-path", + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "gix-tempfile" +version = "12.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8ef376d718b1f5f119b458e21b00fbf576bc9d4e26f8f383d29f5ffe3ba3eaa" +dependencies = [ + "gix-fs", + "libc", + "once_cell", + "parking_lot", + "signal-hook", + "signal-hook-registry", + "tempfile", +] + +[[package]] +name = "gix-trace" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f924267408915fddcd558e3f37295cc7d6a3e50f8bd8b606cee0808c3915157e" + +[[package]] +name = "gix-traverse" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65109e445ba7a409b48f34f570a4d7db72eade1dc1bcff81990a490e86c07161" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "smallvec", + "thiserror", +] + +[[package]] +name = "gix-url" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" +checksum = "8f0f17cceb7552a231d1fec690bc2740c346554e3be6f5d2c41dfa809594dc44" dependencies = [ - "cfg-if", - "js-sys", - "libc", - "wasi 0.11.0+wasi-snapshot-preview1", - "wasm-bindgen", + "bstr", + "gix-features", + "gix-path", + "home", + "thiserror", + "url", ] [[package]] -name = "ghash" -version = "0.5.1" +name = "gix-utils" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +checksum = "35192df7fd0fa112263bad8021e2df7167df4cc2a6e6d15892e1e55621d3d4dc" dependencies = [ - "opaque-debug 0.3.1", - "polyval", + "fastrand", + "unicode-normalization", ] [[package]] -name = "gimli" -version = "0.28.1" +name = "gix-validate" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "e39fc6e06044985eac19dd34d474909e517307582e462b2eb4c8fa51b6241545" +dependencies = [ + "bstr", + "thiserror", +] [[package]] name = "glob" @@ -2225,6 +2958,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" + [[package]] name = "hashbrown" version = "0.14.3" @@ -2481,6 +3220,22 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "human-panic" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4f016c89920bbb30951a8405ecacbb4540db5524313b9445736e7e1855cf370" +dependencies = [ + "anstream", + "anstyle", + "backtrace", + "os_info", + "serde", + "serde_derive", + "toml", + "uuid", +] + [[package]] name = "hyper" version = "0.14.28" @@ -2740,6 +3495,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + [[package]] name = "inferno" version = "0.11.19" @@ -2873,6 +3634,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + [[package]] name = "keccak" version = "0.1.5" @@ -3577,6 +4349,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", + "log", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.48.0", ] @@ -3812,6 +4585,39 @@ dependencies = [ "libc", ] +[[package]] +name = "node-tui" +version = "0.1.0" +dependencies = [ + "better-panic", + "clap", + "color-eyre", + "config", + "crossterm", + "derive_deref", + "directories", + "futures", + "human-panic", + "json5", + "lazy_static", + "libc", + "log", + "pretty_assertions", + "ratatui", + "serde", + "serde_json", + "signal-hook", + "sn_service_management", + "strip-ansi-escapes", + "strum", + "tokio", + "tokio-util 0.7.10", + "tracing", + "tracing-error", + "tracing-subscriber", + "vergen", +] + [[package]] name = "nohash-hasher" version = "0.2.0" @@ -4115,6 +4921,12 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "ordered-float" version = "3.9.2" @@ -4124,6 +4936,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-multimap" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ed8acf08e98e744e5384c8bc63ceb0364e68a6854187221c18df61c4797690e" +dependencies = [ + "dlv-list", + "hashbrown 0.13.2", +] + +[[package]] +name = "os_info" +version = "3.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae99c7fa6dd38c7cafe1ec085e804f8f555a2f8659b0dbe03f1f9963a9b51092" +dependencies = [ + "log", + "serde", + "windows-sys 0.52.0", +] + [[package]] name = "overload" version = "0.1.1" @@ -4217,6 +5050,12 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +[[package]] +name = "pathdiff" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" + [[package]] name = "pbkdf2" version = "0.11.0" @@ -4529,6 +5368,16 @@ dependencies = [ "termtree", ] +[[package]] +name = "pretty_assertions" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -4562,6 +5411,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prodash" +version = "28.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744a264d26b88a6a7e37cbad97953fa233b94d585236310bcbc88474b4092d79" + [[package]] name = "prometheus-client" version = "0.22.2" @@ -5022,6 +5877,27 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "ratatui" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a564a852040e82671dc50a37d88f3aa83bbc690dfc6844cfe7a2591620206a80" +dependencies = [ + "bitflags 2.5.0", + "cassowary", + "compact_str", + "crossterm", + "indoc", + "itertools 0.12.1", + "lru 0.12.3", + "paste", + "serde", + "stability", + "strum", + "unicode-segmentation", + "unicode-width", +] + [[package]] name = "rayon" version = "1.10.0" @@ -5291,6 +6167,18 @@ dependencies = [ "serde", ] +[[package]] +name = "ron" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" +dependencies = [ + "base64 0.21.7", + "bitflags 2.5.0", + "serde", + "serde_derive", +] + [[package]] name = "rtnetlink" version = "0.10.1" @@ -5306,6 +6194,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "rust-ini" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e2a3bcec1f113553ef1c88aae6c020a369d03d55b58de9869a0908930385091" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -5615,6 +6513,9 @@ name = "semver" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" +dependencies = [ + "serde", +] [[package]] name = "send_wrapper" @@ -5668,6 +6569,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +dependencies = [ + "serde", +] + [[package]] name = "serde_test" version = "1.0.176" @@ -5739,6 +6649,12 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sha2" version = "0.8.2" @@ -5802,6 +6718,27 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" +[[package]] +name = "signal-hook" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -6443,6 +7380,16 @@ dependencies = [ "der 0.7.9", ] +[[package]] +name = "stability" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ff9eaf853dec4c8802325d8b6d3dffa86cc707fd7a1a4cdbf416e13b061787a" +dependencies = [ + "quote", + "syn 2.0.60", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -6461,6 +7408,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" +[[package]] +name = "strip-ansi-escapes" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ff8ef943b384c414f54aefa961dd2bd853add74ec75e7ac74cf91dba62bcfa" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.10.0" @@ -6640,6 +7596,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "terminal_size" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" +dependencies = [ + "rustix", + "windows-sys 0.48.0", +] + [[package]] name = "termtree" version = "0.4.1" @@ -6913,6 +7879,40 @@ dependencies = [ "tracing", ] +[[package]] +name = "toml" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e40bb779c5187258fd7aad0eb68cb8706a0a81fa712fbea808ab43c4b8374c4" +dependencies = [ + "indexmap 2.2.6", + "serde", + "serde_spanned", + "toml_datetime", + "winnow 0.6.6", +] + [[package]] name = "tonic" version = "0.6.2" @@ -7255,6 +8255,12 @@ version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +[[package]] +name = "unicode-bom" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" + [[package]] name = "unicode-ident" version = "1.0.12" @@ -7393,7 +8399,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e27d6bdd219887a9eadd19e1c34f32e47fa332301184935c6d9bca26f3cca525" dependencies = [ "anyhow", + "cargo_metadata", "cfg-if", + "gix", + "regex", "rustversion", "time", ] @@ -7410,6 +8419,26 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "vte" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" +dependencies = [ + "utf8parse", + "vte_generate_state_changes", +] + +[[package]] +name = "vte_generate_state_changes" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d257817081c7dffcdbab24b9e62d2def62e2ff7d00b1c20062551e6cccc145ff" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "wait-timeout" version = "0.2.0" @@ -7830,6 +8859,24 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + +[[package]] +name = "winnow" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c976aaaa0e1f90dbb21e9587cdaf1d9679a1cde8875c0d6bd83ab96a208352" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" @@ -7928,6 +8975,15 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "yamux" version = "0.12.1" @@ -7959,6 +9015,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + [[package]] name = "yasna" version = "0.5.2" diff --git a/Cargo.toml b/Cargo.toml index c4671d91f1..d3eeb6d517 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "sn_transfers", "test_utils", "token_supplies", + "node-tui" ] [workspace.lints.rust] diff --git a/node-tui/.config/config.json5 b/node-tui/.config/config.json5 new file mode 100644 index 0000000000..c746239314 --- /dev/null +++ b/node-tui/.config/config.json5 @@ -0,0 +1,10 @@ +{ + "keybindings": { + "Home": { + "": "Quit", // Quit the application + "": "Quit", // Another way to quit + "": "Quit", // Yet another way to quit + "": "Suspend" // Suspend the application + }, + } +} diff --git a/node-tui/.envrc b/node-tui/.envrc new file mode 100644 index 0000000000..7a24791ff5 --- /dev/null +++ b/node-tui/.envrc @@ -0,0 +1,3 @@ +export NODE_TUI_CONFIG=`pwd`/.config +export NODE_TUI_DATA=`pwd`/.data +export NODE_TUI_LOG_LEVEL=debug diff --git a/node-tui/.github/workflows/cd.yml b/node-tui/.github/workflows/cd.yml new file mode 100644 index 0000000000..f7c06cd77e --- /dev/null +++ b/node-tui/.github/workflows/cd.yml @@ -0,0 +1,152 @@ +name: CD # Continuous Deployment + +on: + push: + tags: + - '[v]?[0-9]+.[0-9]+.[0-9]+' + +jobs: + publish: + + name: Publishing for ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + matrix: + include: + - os: macos-latest + os-name: macos + target: x86_64-apple-darwin + architecture: x86_64 + binary-postfix: "" + binary-name: node-tui + use-cross: false + - os: macos-latest + os-name: macos + target: aarch64-apple-darwin + architecture: arm64 + binary-postfix: "" + use-cross: false + binary-name: node-tui + - os: ubuntu-latest + os-name: linux + target: x86_64-unknown-linux-gnu + architecture: x86_64 + binary-postfix: "" + use-cross: false + binary-name: node-tui + - os: windows-latest + os-name: windows + target: x86_64-pc-windows-msvc + architecture: x86_64 + binary-postfix: ".exe" + use-cross: false + binary-name: node-tui + - os: ubuntu-latest + os-name: linux + target: aarch64-unknown-linux-gnu + architecture: arm64 + binary-postfix: "" + use-cross: true + binary-name: node-tui + - os: ubuntu-latest + os-name: linux + target: i686-unknown-linux-gnu + architecture: i686 + binary-postfix: "" + use-cross: true + binary-name: node-tui + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Install Rust toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + + target: ${{ matrix.target }} + + profile: minimal + override: true + - uses: Swatinem/rust-cache@v2 + - name: Cargo build + uses: actions-rs/cargo@v1 + with: + command: build + + use-cross: ${{ matrix.use-cross }} + + toolchain: stable + + args: --release --target ${{ matrix.target }} + + + - name: install strip command + shell: bash + run: | + + if [[ ${{ matrix.target }} == aarch64-unknown-linux-gnu ]]; then + + sudo apt update + sudo apt-get install -y binutils-aarch64-linux-gnu + fi + - name: Packaging final binary + shell: bash + run: | + + cd target/${{ matrix.target }}/release + + + ####### reduce binary size by removing debug symbols ####### + + BINARY_NAME=${{ matrix.binary-name }}${{ matrix.binary-postfix }} + if [[ ${{ matrix.target }} == aarch64-unknown-linux-gnu ]]; then + + GCC_PREFIX="aarch64-linux-gnu-" + else + GCC_PREFIX="" + fi + "$GCC_PREFIX"strip $BINARY_NAME + + ########## create tar.gz ########## + + RELEASE_NAME=${{ matrix.binary-name }}-${GITHUB_REF/refs\/tags\//}-${{ matrix.os-name }}-${{ matrix.architecture }} + + tar czvf $RELEASE_NAME.tar.gz $BINARY_NAME + + ########## create sha256 ########## + + if [[ ${{ runner.os }} == 'Windows' ]]; then + + certutil -hashfile $RELEASE_NAME.tar.gz sha256 | grep -E [A-Fa-f0-9]{64} > $RELEASE_NAME.sha256 + else + shasum -a 256 $RELEASE_NAME.tar.gz > $RELEASE_NAME.sha256 + fi + - name: Releasing assets + uses: softprops/action-gh-release@v1 + with: + files: | + + target/${{ matrix.target }}/release/${{ matrix.binary-name }}-*.tar.gz + target/${{ matrix.target }}/release/${{ matrix.binary-name }}-*.sha256 + + env: + + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + + publish-cargo: + name: Publishing to Cargo + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: cargo publish + env: + + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + diff --git a/node-tui/.github/workflows/ci.yml b/node-tui/.github/workflows/ci.yml new file mode 100644 index 0000000000..c64fbee9e3 --- /dev/null +++ b/node-tui/.github/workflows/ci.yml @@ -0,0 +1,63 @@ +name: CI # Continuous Integration + +on: + push: + branches: + - main + pull_request: + +jobs: + + test: + name: Test Suite + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@nightly + - uses: Swatinem/rust-cache@v2 + - name: Run tests + run: cargo test --all-features --workspace + + rustfmt: + name: Rustfmt + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@nightly + with: + components: rustfmt + - uses: Swatinem/rust-cache@v2 + - name: Check formatting + run: cargo fmt --all --check + + clippy: + name: Clippy + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@nightly + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + - name: Clippy check + run: cargo clippy --all-targets --all-features --workspace -- -D warnings + + docs: + name: Docs + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@nightly + - uses: Swatinem/rust-cache@v2 + - name: Check documentation + env: + RUSTDOCFLAGS: -D warnings + run: cargo doc --no-deps --document-private-items --all-features --workspace --examples diff --git a/node-tui/.gitignore b/node-tui/.gitignore new file mode 100644 index 0000000000..8d5254c8e3 --- /dev/null +++ b/node-tui/.gitignore @@ -0,0 +1,2 @@ +/target +.data/*.log diff --git a/node-tui/.rustfmt.toml b/node-tui/.rustfmt.toml new file mode 100644 index 0000000000..491199d2b9 --- /dev/null +++ b/node-tui/.rustfmt.toml @@ -0,0 +1,16 @@ +max_width = 120 +use_small_heuristics = "Max" +empty_item_single_line = false +force_multiline_blocks = true +format_code_in_doc_comments = true +match_block_trailing_comma = true +imports_granularity = "Crate" +normalize_comments = true +normalize_doc_attributes = true +overflow_delimited_expr = true +reorder_impl_items = true +reorder_imports = true +group_imports = "StdExternalCrate" +tab_spaces = 4 +use_field_init_shorthand = true +use_try_shorthand = true diff --git a/node-tui/Cargo.toml b/node-tui/Cargo.toml new file mode 100644 index 0000000000..344996d235 --- /dev/null +++ b/node-tui/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "node-tui" +version = "0.1.0" +edition = "2021" +description = "Terminal interface for autonomi node management" + +authors = ["Josh Wilson "] +build = "build.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +better-panic = "0.3.0" +clap = { version = "4.4.5", features = [ + "derive", + "cargo", + "wrap_help", + "unicode", + "string", + "unstable-styles", +] } +color-eyre = "0.6.2" +config = "0.14.0" +crossterm = { version = "0.27.0", features = ["serde", "event-stream"] } +derive_deref = "1.1.1" +directories = "5.0.1" +futures = "0.3.28" +human-panic = "1.2.0" +json5 = "0.4.1" +lazy_static = "1.4.0" +libc = "0.2.148" +log = "0.4.20" +pretty_assertions = "1.4.0" +ratatui = { version = "0.26.0", features = ["serde", "macros"] } +serde = { version = "1.0.188", features = ["derive"] } +sn_service_management = { verison = "0.2.4", path = "../sn_service_management" } +serde_json = "1.0.107" +signal-hook = "0.3.17" +strip-ansi-escapes = "0.2.0" +strum = { version = "0.26.1", features = ["derive"] } +tokio = { version = "1.32.0", features = ["full"] } +tokio-util = "0.7.9" +tracing = "0.1.37" +tracing-error = "0.2.0" +tracing-subscriber = { version = "0.3.17", features = ["env-filter", "serde"] } + +[build-dependencies] +vergen = { version = "8.2.6", features = ["build", "git", "gitoxide", "cargo"] } diff --git a/node-tui/LICENSE b/node-tui/LICENSE new file mode 100644 index 0000000000..80002c7348 --- /dev/null +++ b/node-tui/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Maidsafe.net + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/node-tui/README.md b/node-tui/README.md new file mode 100644 index 0000000000..9fcf95277b --- /dev/null +++ b/node-tui/README.md @@ -0,0 +1,5 @@ +# node-tui + +[![CI](https://github.com//node-tui/workflows/CI/badge.svg)](https://github.com//node-tui/actions) + +Terminal interface for autonomi node management diff --git a/node-tui/build.rs b/node-tui/build.rs new file mode 100644 index 0000000000..cb965a14a1 --- /dev/null +++ b/node-tui/build.rs @@ -0,0 +1,4 @@ +fn main() -> Result<(), Box> { + vergen::EmitBuilder::builder().all_build().all_git().emit()?; + Ok(()) +} diff --git a/node-tui/rust-toolchain.toml b/node-tui/rust-toolchain.toml new file mode 100644 index 0000000000..5d56faf9ae --- /dev/null +++ b/node-tui/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "nightly" diff --git a/node-tui/src/action.rs b/node-tui/src/action.rs new file mode 100644 index 0000000000..705e492e6d --- /dev/null +++ b/node-tui/src/action.rs @@ -0,0 +1,21 @@ +use std::{fmt, string::ToString}; + +use serde::{ + de::{self, Deserializer, Visitor}, + Deserialize, Serialize, +}; +use strum::Display; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Display, Deserialize)] +pub enum Action { + Tick, + Render, + Resize(u16, u16), + Suspend, + Resume, + StartNode, + Quit, + Refresh, + Error(String), + Help, +} diff --git a/node-tui/src/app.rs b/node-tui/src/app.rs new file mode 100644 index 0000000000..c6c83e0345 --- /dev/null +++ b/node-tui/src/app.rs @@ -0,0 +1,151 @@ +use color_eyre::eyre::Result; +use crossterm::event::KeyEvent; +use ratatui::prelude::Rect; +use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc; + +use crate::{ + action::Action, + components::{fps::FpsCounter, home::Home, Component}, + config::Config, + mode::Mode, + tui, +}; + +pub struct App { + pub config: Config, + pub tick_rate: f64, + pub frame_rate: f64, + pub components: Vec>, + pub should_quit: bool, + pub should_suspend: bool, + pub mode: Mode, + pub last_tick_key_events: Vec, +} + +impl App { + pub fn new(tick_rate: f64, frame_rate: f64) -> Result { + let home = Home::new(); + let fps = FpsCounter::default(); + let config = Config::new()?; + let mode = Mode::Home; + Ok(Self { + tick_rate, + frame_rate, + components: vec![Box::new(home), Box::new(fps)], + should_quit: false, + should_suspend: false, + config, + mode, + last_tick_key_events: Vec::new(), + }) + } + + pub async fn run(&mut self) -> Result<()> { + let (action_tx, mut action_rx) = mpsc::unbounded_channel(); + + let mut tui = tui::Tui::new()?.tick_rate(self.tick_rate).frame_rate(self.frame_rate); + // tui.mouse(true); + tui.enter()?; + + for component in self.components.iter_mut() { + component.register_action_handler(action_tx.clone())?; + } + + for component in self.components.iter_mut() { + component.register_config_handler(self.config.clone())?; + } + + for component in self.components.iter_mut() { + component.init(tui.size()?)?; + } + + loop { + if let Some(e) = tui.next().await { + match e { + tui::Event::Quit => action_tx.send(Action::Quit)?, + tui::Event::Tick => action_tx.send(Action::Tick)?, + tui::Event::Render => action_tx.send(Action::Render)?, + tui::Event::Resize(x, y) => action_tx.send(Action::Resize(x, y))?, + tui::Event::Key(key) => { + if let Some(keymap) = self.config.keybindings.get(&self.mode) { + if let Some(action) = keymap.get(&vec![key]) { + log::info!("Got action: {action:?}"); + action_tx.send(action.clone())?; + } else { + // If the key was not handled as a single key action, + // then consider it for multi-key combinations. + self.last_tick_key_events.push(key); + + // Check for multi-key combinations + if let Some(action) = keymap.get(&self.last_tick_key_events) { + log::info!("Got action: {action:?}"); + action_tx.send(action.clone())?; + } + } + }; + }, + _ => {}, + } + for component in self.components.iter_mut() { + if let Some(action) = component.handle_events(Some(e.clone()))? { + action_tx.send(action)?; + } + } + } + + while let Ok(action) = action_rx.try_recv() { + if action != Action::Tick && action != Action::Render { + log::debug!("{action:?}"); + } + match action { + Action::Tick => { + self.last_tick_key_events.drain(..); + }, + Action::Quit => self.should_quit = true, + Action::Suspend => self.should_suspend = true, + Action::Resume => self.should_suspend = false, + Action::Resize(w, h) => { + tui.resize(Rect::new(0, 0, w, h))?; + tui.draw(|f| { + for component in self.components.iter_mut() { + let r = component.draw(f, f.size()); + if let Err(e) = r { + action_tx.send(Action::Error(format!("Failed to draw: {:?}", e))).unwrap(); + } + } + })?; + }, + Action::Render => { + tui.draw(|f| { + for component in self.components.iter_mut() { + let r = component.draw(f, f.size()); + if let Err(e) = r { + action_tx.send(Action::Error(format!("Failed to draw: {:?}", e))).unwrap(); + } + } + })?; + }, + _ => {}, + } + for component in self.components.iter_mut() { + if let Some(action) = component.update(action.clone())? { + action_tx.send(action)? + }; + } + } + if self.should_suspend { + tui.suspend()?; + action_tx.send(Action::Resume)?; + tui = tui::Tui::new()?.tick_rate(self.tick_rate).frame_rate(self.frame_rate); + // tui.mouse(true); + tui.enter()?; + } else if self.should_quit { + tui.stop()?; + break; + } + } + tui.exit()?; + Ok(()) + } +} diff --git a/node-tui/src/cli.rs b/node-tui/src/cli.rs new file mode 100644 index 0000000000..edaaf35b40 --- /dev/null +++ b/node-tui/src/cli.rs @@ -0,0 +1,27 @@ +use std::path::PathBuf; + +use clap::Parser; + +use crate::utils::version; + +#[derive(Parser, Debug)] +#[command(author, version = version(), about)] +pub struct Cli { + #[arg( + short, + long, + value_name = "FLOAT", + help = "Tick rate, i.e. number of ticks per second", + default_value_t = 1.0 + )] + pub tick_rate: f64, + + #[arg( + short, + long, + value_name = "FLOAT", + help = "Frame rate, i.e. number of frames per second", + default_value_t = 4.0 + )] + pub frame_rate: f64, +} diff --git a/node-tui/src/components.rs b/node-tui/src/components.rs new file mode 100644 index 0000000000..6500f3bc5f --- /dev/null +++ b/node-tui/src/components.rs @@ -0,0 +1,124 @@ +use color_eyre::eyre::Result; +use crossterm::event::{KeyEvent, MouseEvent}; +use ratatui::layout::Rect; +use tokio::sync::mpsc::UnboundedSender; + +use crate::{ + action::Action, + config::Config, + tui::{Event, Frame}, +}; + +pub mod fps; +pub mod home; + +/// `Component` is a trait that represents a visual and interactive element of the user interface. +/// Implementors of this trait can be registered with the main application loop and will be able to receive events, +/// update state, and be rendered on the screen. +pub trait Component { + /// Register an action handler that can send actions for processing if necessary. + /// + /// # Arguments + /// + /// * `tx` - An unbounded sender that can send actions. + /// + /// # Returns + /// + /// * `Result<()>` - An Ok result or an error. + #[allow(unused_variables)] + fn register_action_handler(&mut self, tx: UnboundedSender) -> Result<()> { + Ok(()) + } + /// Register a configuration handler that provides configuration settings if necessary. + /// + /// # Arguments + /// + /// * `config` - Configuration settings. + /// + /// # Returns + /// + /// * `Result<()>` - An Ok result or an error. + #[allow(unused_variables)] + fn register_config_handler(&mut self, config: Config) -> Result<()> { + Ok(()) + } + /// Initialize the component with a specified area if necessary. + /// + /// # Arguments + /// + /// * `area` - Rectangular area to initialize the component within. + /// + /// # Returns + /// + /// * `Result<()>` - An Ok result or an error. + fn init(&mut self, area: Rect) -> Result<()> { + Ok(()) + } + /// Handle incoming events and produce actions if necessary. + /// + /// # Arguments + /// + /// * `event` - An optional event to be processed. + /// + /// # Returns + /// + /// * `Result>` - An action to be processed or none. + fn handle_events(&mut self, event: Option) -> Result> { + let r = match event { + Some(Event::Key(key_event)) => self.handle_key_events(key_event)?, + Some(Event::Mouse(mouse_event)) => self.handle_mouse_events(mouse_event)?, + _ => None, + }; + Ok(r) + } + /// Handle key events and produce actions if necessary. + /// + /// # Arguments + /// + /// * `key` - A key event to be processed. + /// + /// # Returns + /// + /// * `Result>` - An action to be processed or none. + #[allow(unused_variables)] + fn handle_key_events(&mut self, key: KeyEvent) -> Result> { + Ok(None) + } + /// Handle mouse events and produce actions if necessary. + /// + /// # Arguments + /// + /// * `mouse` - A mouse event to be processed. + /// + /// # Returns + /// + /// * `Result>` - An action to be processed or none. + #[allow(unused_variables)] + fn handle_mouse_events(&mut self, mouse: MouseEvent) -> Result> { + Ok(None) + } + /// Update the state of the component based on a received action. (REQUIRED) + /// + /// # Arguments + /// + /// * `action` - An action that may modify the state of the component. + /// + /// # Returns + /// + /// * `Result>` - An action to be processed or none. + #[allow(unused_variables)] + fn update(&mut self, action: Action) -> Result> { + Ok(None) + } + /// Render the component on the screen. (REQUIRED) + /// + /// # Arguments + /// + /// * `f` - A frame used for rendering. + /// * `area` - The area in which the component should be drawn. + /// + /// # Returns + /// + /// * `Result<()>` - An Ok result or an error. + fn draw(&mut self, f: &mut Frame<'_>, area: Rect) -> Result<()>; +} diff --git a/node-tui/src/components/fps.rs b/node-tui/src/components/fps.rs new file mode 100644 index 0000000000..a775bc483f --- /dev/null +++ b/node-tui/src/components/fps.rs @@ -0,0 +1,90 @@ +use std::time::Instant; + +use color_eyre::eyre::Result; +use ratatui::{prelude::*, widgets::*}; + +use super::Component; +use crate::{action::Action, tui::Frame}; + +#[derive(Debug, Clone, PartialEq)] +pub struct FpsCounter { + app_start_time: Instant, + app_frames: u32, + app_fps: f64, + + render_start_time: Instant, + render_frames: u32, + render_fps: f64, +} + +impl Default for FpsCounter { + fn default() -> Self { + Self::new() + } +} + +impl FpsCounter { + pub fn new() -> Self { + Self { + app_start_time: Instant::now(), + app_frames: 0, + app_fps: 0.0, + render_start_time: Instant::now(), + render_frames: 0, + render_fps: 0.0, + } + } + + fn app_tick(&mut self) -> Result<()> { + self.app_frames += 1; + let now = Instant::now(); + let elapsed = (now - self.app_start_time).as_secs_f64(); + if elapsed >= 1.0 { + self.app_fps = self.app_frames as f64 / elapsed; + self.app_start_time = now; + self.app_frames = 0; + } + Ok(()) + } + + fn render_tick(&mut self) -> Result<()> { + self.render_frames += 1; + let now = Instant::now(); + let elapsed = (now - self.render_start_time).as_secs_f64(); + if elapsed >= 1.0 { + self.render_fps = self.render_frames as f64 / elapsed; + self.render_start_time = now; + self.render_frames = 0; + } + Ok(()) + } +} + +impl Component for FpsCounter { + fn update(&mut self, action: Action) -> Result> { + if let Action::Tick = action { + self.app_tick()? + }; + if let Action::Render = action { + self.render_tick()? + }; + Ok(None) + } + + fn draw(&mut self, f: &mut Frame<'_>, rect: Rect) -> Result<()> { + let rects = Layout::default() + .direction(Direction::Vertical) + .constraints(vec![ + Constraint::Length(1), // first row + Constraint::Min(0), + ]) + .split(rect); + + let rect = rects[0]; + + let s = format!("{:.2} ticks per sec (app) {:.2} frames per sec (render)", self.app_fps, self.render_fps); + let block = Block::default().title(block::Title::from(s.dim()).alignment(Alignment::Right)); + f.render_widget(block, rect); + Ok(()) + } +} diff --git a/node-tui/src/components/home.rs b/node-tui/src/components/home.rs new file mode 100644 index 0000000000..1af0c461cb --- /dev/null +++ b/node-tui/src/components/home.rs @@ -0,0 +1,100 @@ +use std::{collections::HashMap, time::Duration}; + +use color_eyre::eyre::Result; +use crossterm::event::{KeyCode, KeyEvent}; +use ratatui::{prelude::*, widgets::*}; +use serde::{Deserialize, Serialize}; +use sn_service_management::{get_local_node_registry_path, NodeRegistry, ServiceStatus}; +use tokio::sync::mpsc::UnboundedSender; + +use super::{Component, Frame}; +use crate::{ + action::Action, + config::{Config, KeyBindings}, +}; + +#[derive(Default)] +pub struct Home { + command_tx: Option>, + config: Config, + node_registry: Option, +} + +impl Home { + pub fn new() -> Self { + Self::default() + } +} + +impl Component for Home { + fn register_action_handler(&mut self, tx: UnboundedSender) -> Result<()> { + self.command_tx = Some(tx); + Ok(()) + } + + fn register_config_handler(&mut self, config: Config) -> Result<()> { + self.config = config; + Ok(()) + } + + fn update(&mut self, action: Action) -> Result> { + match action { + Action::Tick => { + let local_node_registry = NodeRegistry::load(&get_local_node_registry_path()?)?; + + if !local_node_registry.nodes.is_empty() { + self.node_registry = Some(local_node_registry); + } else { + self.node_registry = None; + } + }, + Action::StartNode => { + let local_node_registry = NodeRegistry::load(&get_local_node_registry_path()?)?; + }, + _ => {}, + } + Ok(None) + } + + fn draw(&mut self, f: &mut Frame<'_>, area: Rect) -> Result<()> { + if let Some(registry) = &self.node_registry { + let nodes: Vec<_> = + registry + .to_status_summary() + .nodes + .iter() + .filter_map(|n| { + if let ServiceStatus::Running = n.status { + n.peer_id.map(|p| p.to_string()) + } else { + None + } + }) + .collect(); + + if !nodes.is_empty() { + let mut list = List::new(nodes); + let home_layout = + Layout::new(Direction::Horizontal, [Constraint::Min(5), Constraint::Min(0), Constraint::Length(1)]) + .split(area); + + f.render_widget( + Paragraph::new("TODO: All Node Stats") + .block(Block::default().title("Autonomi Node Runner").borders(Borders::ALL)), + home_layout[0], + ); + f.render_widget( + list.block(Block::default().title("Running nodes").borders(Borders::ALL)), + home_layout[1], + ); + } + } else { + f.render_widget( + Paragraph::new("No nodes running") + .block(Block::default().title("Autonomi Node Runner").borders(Borders::ALL)), + area, + ) + } + Ok(()) + } +} diff --git a/node-tui/src/config.rs b/node-tui/src/config.rs new file mode 100644 index 0000000000..110533bc06 --- /dev/null +++ b/node-tui/src/config.rs @@ -0,0 +1,504 @@ +use std::{collections::HashMap, fmt, path::PathBuf}; + +use color_eyre::eyre::Result; +use config::Value; +use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; +use derive_deref::{Deref, DerefMut}; +use ratatui::style::{Color, Modifier, Style}; +use serde::{ + de::{self, Deserializer, MapAccess, Visitor}, + Deserialize, Serialize, +}; +use serde_json::Value as JsonValue; + +use crate::{action::Action, mode::Mode}; + +const CONFIG: &str = include_str!("../.config/config.json5"); + +#[derive(Clone, Debug, Deserialize, Default)] +pub struct AppConfig { + #[serde(default)] + pub _data_dir: PathBuf, + #[serde(default)] + pub _config_dir: PathBuf, +} + +#[derive(Clone, Debug, Default, Deserialize)] +pub struct Config { + #[serde(default, flatten)] + pub config: AppConfig, + #[serde(default)] + pub keybindings: KeyBindings, + #[serde(default)] + pub styles: Styles, +} + +impl Config { + pub fn new() -> Result { + let default_config: Config = json5::from_str(CONFIG).unwrap(); + let data_dir = crate::utils::get_data_dir(); + let config_dir = crate::utils::get_config_dir(); + let mut builder = config::Config::builder() + .set_default("_data_dir", data_dir.to_str().unwrap())? + .set_default("_config_dir", config_dir.to_str().unwrap())?; + + let config_files = [ + ("config.json5", config::FileFormat::Json5), + ("config.json", config::FileFormat::Json), + ("config.yaml", config::FileFormat::Yaml), + ("config.toml", config::FileFormat::Toml), + ("config.ini", config::FileFormat::Ini), + ]; + let mut found_config = false; + for (file, format) in &config_files { + builder = builder.add_source(config::File::from(config_dir.join(file)).format(*format).required(false)); + if config_dir.join(file).exists() { + found_config = true + } + } + if !found_config { + log::error!("No configuration file found. Application may not behave as expected"); + } + + let mut cfg: Self = builder.build()?.try_deserialize()?; + + for (mode, default_bindings) in default_config.keybindings.iter() { + let user_bindings = cfg.keybindings.entry(*mode).or_default(); + for (key, cmd) in default_bindings.iter() { + user_bindings.entry(key.clone()).or_insert_with(|| cmd.clone()); + } + } + for (mode, default_styles) in default_config.styles.iter() { + let user_styles = cfg.styles.entry(*mode).or_default(); + for (style_key, style) in default_styles.iter() { + user_styles.entry(style_key.clone()).or_insert_with(|| style.clone()); + } + } + + Ok(cfg) + } +} + +#[derive(Clone, Debug, Default, Deref, DerefMut)] +pub struct KeyBindings(pub HashMap, Action>>); + +impl<'de> Deserialize<'de> for KeyBindings { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let parsed_map = HashMap::>::deserialize(deserializer)?; + + let keybindings = parsed_map + .into_iter() + .map(|(mode, inner_map)| { + let converted_inner_map = + inner_map.into_iter().map(|(key_str, cmd)| (parse_key_sequence(&key_str).unwrap(), cmd)).collect(); + (mode, converted_inner_map) + }) + .collect(); + + Ok(KeyBindings(keybindings)) + } +} + +fn parse_key_event(raw: &str) -> Result { + let raw_lower = raw.to_ascii_lowercase(); + let (remaining, modifiers) = extract_modifiers(&raw_lower); + parse_key_code_with_modifiers(remaining, modifiers) +} + +fn extract_modifiers(raw: &str) -> (&str, KeyModifiers) { + let mut modifiers = KeyModifiers::empty(); + let mut current = raw; + + loop { + match current { + rest if rest.starts_with("ctrl-") => { + modifiers.insert(KeyModifiers::CONTROL); + current = &rest[5..]; + }, + rest if rest.starts_with("alt-") => { + modifiers.insert(KeyModifiers::ALT); + current = &rest[4..]; + }, + rest if rest.starts_with("shift-") => { + modifiers.insert(KeyModifiers::SHIFT); + current = &rest[6..]; + }, + _ => break, // break out of the loop if no known prefix is detected + }; + } + + (current, modifiers) +} + +fn parse_key_code_with_modifiers(raw: &str, mut modifiers: KeyModifiers) -> Result { + let c = match raw { + "esc" => KeyCode::Esc, + "enter" => KeyCode::Enter, + "left" => KeyCode::Left, + "right" => KeyCode::Right, + "up" => KeyCode::Up, + "down" => KeyCode::Down, + "home" => KeyCode::Home, + "end" => KeyCode::End, + "pageup" => KeyCode::PageUp, + "pagedown" => KeyCode::PageDown, + "backtab" => { + modifiers.insert(KeyModifiers::SHIFT); + KeyCode::BackTab + }, + "backspace" => KeyCode::Backspace, + "delete" => KeyCode::Delete, + "insert" => KeyCode::Insert, + "f1" => KeyCode::F(1), + "f2" => KeyCode::F(2), + "f3" => KeyCode::F(3), + "f4" => KeyCode::F(4), + "f5" => KeyCode::F(5), + "f6" => KeyCode::F(6), + "f7" => KeyCode::F(7), + "f8" => KeyCode::F(8), + "f9" => KeyCode::F(9), + "f10" => KeyCode::F(10), + "f11" => KeyCode::F(11), + "f12" => KeyCode::F(12), + "space" => KeyCode::Char(' '), + "hyphen" => KeyCode::Char('-'), + "minus" => KeyCode::Char('-'), + "tab" => KeyCode::Tab, + c if c.len() == 1 => { + let mut c = c.chars().next().unwrap(); + if modifiers.contains(KeyModifiers::SHIFT) { + c = c.to_ascii_uppercase(); + } + KeyCode::Char(c) + }, + _ => return Err(format!("Unable to parse {raw}")), + }; + Ok(KeyEvent::new(c, modifiers)) +} + +pub fn key_event_to_string(key_event: &KeyEvent) -> String { + let char; + let key_code = match key_event.code { + KeyCode::Backspace => "backspace", + KeyCode::Enter => "enter", + KeyCode::Left => "left", + KeyCode::Right => "right", + KeyCode::Up => "up", + KeyCode::Down => "down", + KeyCode::Home => "home", + KeyCode::End => "end", + KeyCode::PageUp => "pageup", + KeyCode::PageDown => "pagedown", + KeyCode::Tab => "tab", + KeyCode::BackTab => "backtab", + KeyCode::Delete => "delete", + KeyCode::Insert => "insert", + KeyCode::F(c) => { + char = format!("f({c})"); + &char + }, + KeyCode::Char(c) if c == ' ' => "space", + KeyCode::Char(c) => { + char = c.to_string(); + &char + }, + KeyCode::Esc => "esc", + KeyCode::Null => "", + KeyCode::CapsLock => "", + KeyCode::Menu => "", + KeyCode::ScrollLock => "", + KeyCode::Media(_) => "", + KeyCode::NumLock => "", + KeyCode::PrintScreen => "", + KeyCode::Pause => "", + KeyCode::KeypadBegin => "", + KeyCode::Modifier(_) => "", + }; + + let mut modifiers = Vec::with_capacity(3); + + if key_event.modifiers.intersects(KeyModifiers::CONTROL) { + modifiers.push("ctrl"); + } + + if key_event.modifiers.intersects(KeyModifiers::SHIFT) { + modifiers.push("shift"); + } + + if key_event.modifiers.intersects(KeyModifiers::ALT) { + modifiers.push("alt"); + } + + let mut key = modifiers.join("-"); + + if !key.is_empty() { + key.push('-'); + } + key.push_str(key_code); + + key +} + +pub fn parse_key_sequence(raw: &str) -> Result, String> { + if raw.chars().filter(|c| *c == '>').count() != raw.chars().filter(|c| *c == '<').count() { + return Err(format!("Unable to parse `{}`", raw)); + } + let raw = if !raw.contains("><") { + let raw = raw.strip_prefix('<').unwrap_or(raw); + let raw = raw.strip_prefix('>').unwrap_or(raw); + raw + } else { + raw + }; + let sequences = raw + .split("><") + .map(|seq| { + if let Some(s) = seq.strip_prefix('<') { + s + } else if let Some(s) = seq.strip_suffix('>') { + s + } else { + seq + } + }) + .collect::>(); + + sequences.into_iter().map(parse_key_event).collect() +} + +#[derive(Clone, Debug, Default, Deref, DerefMut)] +pub struct Styles(pub HashMap>); + +impl<'de> Deserialize<'de> for Styles { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let parsed_map = HashMap::>::deserialize(deserializer)?; + + let styles = parsed_map + .into_iter() + .map(|(mode, inner_map)| { + let converted_inner_map = + inner_map.into_iter().map(|(str, style)| (str, parse_style(&style))).collect(); + (mode, converted_inner_map) + }) + .collect(); + + Ok(Styles(styles)) + } +} + +pub fn parse_style(line: &str) -> Style { + let (foreground, background) = line.split_at(line.to_lowercase().find("on ").unwrap_or(line.len())); + let foreground = process_color_string(foreground); + let background = process_color_string(&background.replace("on ", "")); + + let mut style = Style::default(); + if let Some(fg) = parse_color(&foreground.0) { + style = style.fg(fg); + } + if let Some(bg) = parse_color(&background.0) { + style = style.bg(bg); + } + style = style.add_modifier(foreground.1 | background.1); + style +} + +fn process_color_string(color_str: &str) -> (String, Modifier) { + let color = color_str + .replace("grey", "gray") + .replace("bright ", "") + .replace("bold ", "") + .replace("underline ", "") + .replace("inverse ", ""); + + let mut modifiers = Modifier::empty(); + if color_str.contains("underline") { + modifiers |= Modifier::UNDERLINED; + } + if color_str.contains("bold") { + modifiers |= Modifier::BOLD; + } + if color_str.contains("inverse") { + modifiers |= Modifier::REVERSED; + } + + (color, modifiers) +} + +fn parse_color(s: &str) -> Option { + let s = s.trim_start(); + let s = s.trim_end(); + if s.contains("bright color") { + let s = s.trim_start_matches("bright "); + let c = s.trim_start_matches("color").parse::().unwrap_or_default(); + Some(Color::Indexed(c.wrapping_shl(8))) + } else if s.contains("color") { + let c = s.trim_start_matches("color").parse::().unwrap_or_default(); + Some(Color::Indexed(c)) + } else if s.contains("gray") { + let c = 232 + s.trim_start_matches("gray").parse::().unwrap_or_default(); + Some(Color::Indexed(c)) + } else if s.contains("rgb") { + let red = (s.as_bytes()[3] as char).to_digit(10).unwrap_or_default() as u8; + let green = (s.as_bytes()[4] as char).to_digit(10).unwrap_or_default() as u8; + let blue = (s.as_bytes()[5] as char).to_digit(10).unwrap_or_default() as u8; + let c = 16 + red * 36 + green * 6 + blue; + Some(Color::Indexed(c)) + } else if s == "bold black" { + Some(Color::Indexed(8)) + } else if s == "bold red" { + Some(Color::Indexed(9)) + } else if s == "bold green" { + Some(Color::Indexed(10)) + } else if s == "bold yellow" { + Some(Color::Indexed(11)) + } else if s == "bold blue" { + Some(Color::Indexed(12)) + } else if s == "bold magenta" { + Some(Color::Indexed(13)) + } else if s == "bold cyan" { + Some(Color::Indexed(14)) + } else if s == "bold white" { + Some(Color::Indexed(15)) + } else if s == "black" { + Some(Color::Indexed(0)) + } else if s == "red" { + Some(Color::Indexed(1)) + } else if s == "green" { + Some(Color::Indexed(2)) + } else if s == "yellow" { + Some(Color::Indexed(3)) + } else if s == "blue" { + Some(Color::Indexed(4)) + } else if s == "magenta" { + Some(Color::Indexed(5)) + } else if s == "cyan" { + Some(Color::Indexed(6)) + } else if s == "white" { + Some(Color::Indexed(7)) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use pretty_assertions::assert_eq; + + use super::*; + + #[test] + fn test_parse_style_default() { + let style = parse_style(""); + assert_eq!(style, Style::default()); + } + + #[test] + fn test_parse_style_foreground() { + let style = parse_style("red"); + assert_eq!(style.fg, Some(Color::Indexed(1))); + } + + #[test] + fn test_parse_style_background() { + let style = parse_style("on blue"); + assert_eq!(style.bg, Some(Color::Indexed(4))); + } + + #[test] + fn test_parse_style_modifiers() { + let style = parse_style("underline red on blue"); + assert_eq!(style.fg, Some(Color::Indexed(1))); + assert_eq!(style.bg, Some(Color::Indexed(4))); + } + + #[test] + fn test_process_color_string() { + let (color, modifiers) = process_color_string("underline bold inverse gray"); + assert_eq!(color, "gray"); + assert!(modifiers.contains(Modifier::UNDERLINED)); + assert!(modifiers.contains(Modifier::BOLD)); + assert!(modifiers.contains(Modifier::REVERSED)); + } + + #[test] + fn test_parse_color_rgb() { + let color = parse_color("rgb123"); + let expected = 16 + 1 * 36 + 2 * 6 + 3; + assert_eq!(color, Some(Color::Indexed(expected))); + } + + #[test] + fn test_parse_color_unknown() { + let color = parse_color("unknown"); + assert_eq!(color, None); + } + + #[test] + fn test_config() -> Result<()> { + let c = Config::new()?; + assert_eq!( + c.keybindings.get(&Mode::Home).unwrap().get(&parse_key_sequence("").unwrap_or_default()).unwrap(), + &Action::Quit + ); + Ok(()) + } + + #[test] + fn test_simple_keys() { + assert_eq!(parse_key_event("a").unwrap(), KeyEvent::new(KeyCode::Char('a'), KeyModifiers::empty())); + + assert_eq!(parse_key_event("enter").unwrap(), KeyEvent::new(KeyCode::Enter, KeyModifiers::empty())); + + assert_eq!(parse_key_event("esc").unwrap(), KeyEvent::new(KeyCode::Esc, KeyModifiers::empty())); + } + + #[test] + fn test_with_modifiers() { + assert_eq!(parse_key_event("ctrl-a").unwrap(), KeyEvent::new(KeyCode::Char('a'), KeyModifiers::CONTROL)); + + assert_eq!(parse_key_event("alt-enter").unwrap(), KeyEvent::new(KeyCode::Enter, KeyModifiers::ALT)); + + assert_eq!(parse_key_event("shift-esc").unwrap(), KeyEvent::new(KeyCode::Esc, KeyModifiers::SHIFT)); + } + + #[test] + fn test_multiple_modifiers() { + assert_eq!( + parse_key_event("ctrl-alt-a").unwrap(), + KeyEvent::new(KeyCode::Char('a'), KeyModifiers::CONTROL | KeyModifiers::ALT) + ); + + assert_eq!( + parse_key_event("ctrl-shift-enter").unwrap(), + KeyEvent::new(KeyCode::Enter, KeyModifiers::CONTROL | KeyModifiers::SHIFT) + ); + } + + #[test] + fn test_reverse_multiple_modifiers() { + assert_eq!( + key_event_to_string(&KeyEvent::new(KeyCode::Char('a'), KeyModifiers::CONTROL | KeyModifiers::ALT)), + "ctrl-alt-a".to_string() + ); + } + + #[test] + fn test_invalid_keys() { + assert!(parse_key_event("invalid-key").is_err()); + assert!(parse_key_event("ctrl-invalid-key").is_err()); + } + + #[test] + fn test_case_insensitivity() { + assert_eq!(parse_key_event("CTRL-a").unwrap(), KeyEvent::new(KeyCode::Char('a'), KeyModifiers::CONTROL)); + + assert_eq!(parse_key_event("AlT-eNtEr").unwrap(), KeyEvent::new(KeyCode::Enter, KeyModifiers::ALT)); + } +} diff --git a/node-tui/src/main.rs b/node-tui/src/main.rs new file mode 100644 index 0000000000..18f66847db --- /dev/null +++ b/node-tui/src/main.rs @@ -0,0 +1,41 @@ +/// initially generated from: https://github.com/ratatui-org/templates/blob/main/component/README.md +/// +pub mod action; +pub mod app; +pub mod cli; +pub mod components; +pub mod config; +pub mod mode; +pub mod tui; +pub mod utils; + +use clap::Parser; +use cli::Cli; +use color_eyre::eyre::Result; + +use crate::{ + app::App, + utils::{initialize_logging, initialize_panic_handler, version}, +}; + +async fn tokio_main() -> Result<()> { + initialize_logging()?; + + initialize_panic_handler()?; + + let args = Cli::parse(); + let mut app = App::new(args.tick_rate, args.frame_rate)?; + app.run().await?; + + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<()> { + if let Err(e) = tokio_main().await { + eprintln!("{} error: Something went wrong", env!("CARGO_PKG_NAME")); + Err(e) + } else { + Ok(()) + } +} diff --git a/node-tui/src/mode.rs b/node-tui/src/mode.rs new file mode 100644 index 0000000000..1775f94813 --- /dev/null +++ b/node-tui/src/mode.rs @@ -0,0 +1,7 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Mode { + #[default] + Home, +} diff --git a/node-tui/src/tui.rs b/node-tui/src/tui.rs new file mode 100644 index 0000000000..88da791ef4 --- /dev/null +++ b/node-tui/src/tui.rs @@ -0,0 +1,239 @@ +use std::{ + ops::{Deref, DerefMut}, + time::Duration, +}; + +use color_eyre::eyre::Result; +use crossterm::{ + cursor, + event::{ + DisableBracketedPaste, DisableMouseCapture, EnableBracketedPaste, EnableMouseCapture, Event as CrosstermEvent, + KeyEvent, KeyEventKind, MouseEvent, + }, + terminal::{EnterAlternateScreen, LeaveAlternateScreen}, +}; +use futures::{FutureExt, StreamExt}; +use ratatui::backend::CrosstermBackend as Backend; +use serde::{Deserialize, Serialize}; +use tokio::{ + sync::mpsc::{self, UnboundedReceiver, UnboundedSender}, + task::JoinHandle, +}; +use tokio_util::sync::CancellationToken; + +pub type IO = std::io::Stdout; +pub fn io() -> IO { + std::io::stdout() +} +pub type Frame<'a> = ratatui::Frame<'a>; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum Event { + Init, + Quit, + Error, + Closed, + Tick, + Render, + FocusGained, + FocusLost, + Paste(String), + Key(KeyEvent), + Mouse(MouseEvent), + Resize(u16, u16), +} + +pub struct Tui { + pub terminal: ratatui::Terminal>, + pub task: JoinHandle<()>, + pub cancellation_token: CancellationToken, + pub event_rx: UnboundedReceiver, + pub event_tx: UnboundedSender, + pub frame_rate: f64, + pub tick_rate: f64, + pub mouse: bool, + pub paste: bool, +} + +impl Tui { + pub fn new() -> Result { + let tick_rate = 4.0; + let frame_rate = 60.0; + let terminal = ratatui::Terminal::new(Backend::new(io()))?; + let (event_tx, event_rx) = mpsc::unbounded_channel(); + let cancellation_token = CancellationToken::new(); + let task = tokio::spawn(async {}); + let mouse = false; + let paste = false; + Ok(Self { terminal, task, cancellation_token, event_rx, event_tx, frame_rate, tick_rate, mouse, paste }) + } + + pub fn tick_rate(mut self, tick_rate: f64) -> Self { + self.tick_rate = tick_rate; + self + } + + pub fn frame_rate(mut self, frame_rate: f64) -> Self { + self.frame_rate = frame_rate; + self + } + + pub fn mouse(mut self, mouse: bool) -> Self { + self.mouse = mouse; + self + } + + pub fn paste(mut self, paste: bool) -> Self { + self.paste = paste; + self + } + + pub fn start(&mut self) { + let tick_delay = std::time::Duration::from_secs_f64(1.0 / self.tick_rate); + let render_delay = std::time::Duration::from_secs_f64(1.0 / self.frame_rate); + self.cancel(); + self.cancellation_token = CancellationToken::new(); + let _cancellation_token = self.cancellation_token.clone(); + let _event_tx = self.event_tx.clone(); + self.task = tokio::spawn(async move { + let mut reader = crossterm::event::EventStream::new(); + let mut tick_interval = tokio::time::interval(tick_delay); + let mut render_interval = tokio::time::interval(render_delay); + _event_tx.send(Event::Init).unwrap(); + loop { + let tick_delay = tick_interval.tick(); + let render_delay = render_interval.tick(); + let crossterm_event = reader.next().fuse(); + tokio::select! { + _ = _cancellation_token.cancelled() => { + break; + } + maybe_event = crossterm_event => { + match maybe_event { + Some(Ok(evt)) => { + match evt { + CrosstermEvent::Key(key) => { + if key.kind == KeyEventKind::Press { + _event_tx.send(Event::Key(key)).unwrap(); + } + }, + CrosstermEvent::Mouse(mouse) => { + _event_tx.send(Event::Mouse(mouse)).unwrap(); + }, + CrosstermEvent::Resize(x, y) => { + _event_tx.send(Event::Resize(x, y)).unwrap(); + }, + CrosstermEvent::FocusLost => { + _event_tx.send(Event::FocusLost).unwrap(); + }, + CrosstermEvent::FocusGained => { + _event_tx.send(Event::FocusGained).unwrap(); + }, + CrosstermEvent::Paste(s) => { + _event_tx.send(Event::Paste(s)).unwrap(); + }, + } + } + Some(Err(_)) => { + _event_tx.send(Event::Error).unwrap(); + } + None => {}, + } + }, + _ = tick_delay => { + _event_tx.send(Event::Tick).unwrap(); + }, + _ = render_delay => { + _event_tx.send(Event::Render).unwrap(); + }, + } + } + }); + } + + pub fn stop(&self) -> Result<()> { + self.cancel(); + let mut counter = 0; + while !self.task.is_finished() { + std::thread::sleep(Duration::from_millis(1)); + counter += 1; + if counter > 50 { + self.task.abort(); + } + if counter > 100 { + log::error!("Failed to abort task in 100 milliseconds for unknown reason"); + break; + } + } + Ok(()) + } + + pub fn enter(&mut self) -> Result<()> { + crossterm::terminal::enable_raw_mode()?; + crossterm::execute!(io(), EnterAlternateScreen, cursor::Hide)?; + if self.mouse { + crossterm::execute!(io(), EnableMouseCapture)?; + } + if self.paste { + crossterm::execute!(io(), EnableBracketedPaste)?; + } + self.start(); + Ok(()) + } + + pub fn exit(&mut self) -> Result<()> { + self.stop()?; + if crossterm::terminal::is_raw_mode_enabled()? { + self.flush()?; + if self.paste { + crossterm::execute!(io(), DisableBracketedPaste)?; + } + if self.mouse { + crossterm::execute!(io(), DisableMouseCapture)?; + } + crossterm::execute!(io(), LeaveAlternateScreen, cursor::Show)?; + crossterm::terminal::disable_raw_mode()?; + } + Ok(()) + } + + pub fn cancel(&self) { + self.cancellation_token.cancel(); + } + + pub fn suspend(&mut self) -> Result<()> { + self.exit()?; + #[cfg(not(windows))] + signal_hook::low_level::raise(signal_hook::consts::signal::SIGTSTP)?; + Ok(()) + } + + pub fn resume(&mut self) -> Result<()> { + self.enter()?; + Ok(()) + } + + pub async fn next(&mut self) -> Option { + self.event_rx.recv().await + } +} + +impl Deref for Tui { + type Target = ratatui::Terminal>; + + fn deref(&self) -> &Self::Target { + &self.terminal + } +} + +impl DerefMut for Tui { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.terminal + } +} + +impl Drop for Tui { + fn drop(&mut self) { + self.exit().unwrap(); + } +} diff --git a/node-tui/src/utils.rs b/node-tui/src/utils.rs new file mode 100644 index 0000000000..8798274a7a --- /dev/null +++ b/node-tui/src/utils.rs @@ -0,0 +1,161 @@ +use std::path::PathBuf; + +use color_eyre::eyre::Result; +use directories::ProjectDirs; +use lazy_static::lazy_static; +use tracing::error; +use tracing_error::ErrorLayer; +use tracing_subscriber::{self, prelude::__tracing_subscriber_SubscriberExt, util::SubscriberInitExt, Layer}; + +const VERSION_MESSAGE: &str = + concat!(env!("CARGO_PKG_VERSION"), "-", env!("VERGEN_GIT_DESCRIBE"), " (", env!("VERGEN_BUILD_DATE"), ")"); + +lazy_static! { + pub static ref PROJECT_NAME: String = env!("CARGO_CRATE_NAME").to_uppercase().to_string(); + pub static ref DATA_FOLDER: Option = + std::env::var(format!("{}_DATA", PROJECT_NAME.clone())).ok().map(PathBuf::from); + pub static ref CONFIG_FOLDER: Option = + std::env::var(format!("{}_CONFIG", PROJECT_NAME.clone())).ok().map(PathBuf::from); + pub static ref LOG_ENV: String = format!("{}_LOGLEVEL", PROJECT_NAME.clone()); + pub static ref LOG_FILE: String = format!("{}.log", env!("CARGO_PKG_NAME")); +} + +fn project_directory() -> Option { + ProjectDirs::from("com", "maidsafe", env!("CARGO_PKG_NAME")) +} + +pub fn initialize_panic_handler() -> Result<()> { + let (panic_hook, eyre_hook) = color_eyre::config::HookBuilder::default() + .panic_section(format!("This is a bug. Consider reporting it at {}", env!("CARGO_PKG_REPOSITORY"))) + .capture_span_trace_by_default(false) + .display_location_section(false) + .display_env_section(false) + .into_hooks(); + eyre_hook.install()?; + std::panic::set_hook(Box::new(move |panic_info| { + if let Ok(mut t) = crate::tui::Tui::new() { + if let Err(r) = t.exit() { + error!("Unable to exit Terminal: {:?}", r); + } + } + + #[cfg(not(debug_assertions))] + { + use human_panic::{handle_dump, print_msg, Metadata}; + let meta = Metadata { + version: env!("CARGO_PKG_VERSION").into(), + name: env!("CARGO_PKG_NAME").into(), + authors: env!("CARGO_PKG_AUTHORS").replace(':', ", ").into(), + homepage: env!("CARGO_PKG_HOMEPAGE").into(), + }; + + let file_path = handle_dump(&meta, panic_info); + // prints human-panic message + print_msg(file_path, &meta).expect("human-panic: printing error message to console failed"); + eprintln!("{}", panic_hook.panic_report(panic_info)); // prints color-eyre stack trace to stderr + } + let msg = format!("{}", panic_hook.panic_report(panic_info)); + log::error!("Error: {}", strip_ansi_escapes::strip_str(msg)); + + #[cfg(debug_assertions)] + { + // Better Panic stacktrace that is only enabled when debugging. + better_panic::Settings::auto() + .most_recent_first(false) + .lineno_suffix(true) + .verbosity(better_panic::Verbosity::Full) + .create_panic_handler()(panic_info); + } + + std::process::exit(libc::EXIT_FAILURE); + })); + Ok(()) +} + +pub fn get_data_dir() -> PathBuf { + let directory = if let Some(s) = DATA_FOLDER.clone() { + s + } else if let Some(proj_dirs) = project_directory() { + proj_dirs.data_local_dir().to_path_buf() + } else { + PathBuf::from(".").join(".data") + }; + directory +} + +pub fn get_config_dir() -> PathBuf { + let directory = if let Some(s) = CONFIG_FOLDER.clone() { + s + } else if let Some(proj_dirs) = project_directory() { + proj_dirs.config_local_dir().to_path_buf() + } else { + PathBuf::from(".").join(".config") + }; + directory +} + +pub fn initialize_logging() -> Result<()> { + let directory = get_data_dir(); + std::fs::create_dir_all(directory.clone())?; + let log_path = directory.join(LOG_FILE.clone()); + let log_file = std::fs::File::create(log_path)?; + std::env::set_var( + "RUST_LOG", + std::env::var("RUST_LOG") + .or_else(|_| std::env::var(LOG_ENV.clone())) + .unwrap_or_else(|_| format!("{}=info", env!("CARGO_CRATE_NAME"))), + ); + let file_subscriber = tracing_subscriber::fmt::layer() + .with_file(true) + .with_line_number(true) + .with_writer(log_file) + .with_target(false) + .with_ansi(false) + .with_filter(tracing_subscriber::filter::EnvFilter::from_default_env()); + tracing_subscriber::registry().with(file_subscriber).with(ErrorLayer::default()).init(); + Ok(()) +} + +/// Similar to the `std::dbg!` macro, but generates `tracing` events rather +/// than printing to stdout. +/// +/// By default, the verbosity level for the generated events is `DEBUG`, but +/// this can be customized. +#[macro_export] +macro_rules! trace_dbg { + (target: $target:expr, level: $level:expr, $ex:expr) => {{ + match $ex { + value => { + tracing::event!(target: $target, $level, ?value, stringify!($ex)); + value + } + } + }}; + (level: $level:expr, $ex:expr) => { + trace_dbg!(target: module_path!(), level: $level, $ex) + }; + (target: $target:expr, $ex:expr) => { + trace_dbg!(target: $target, level: tracing::Level::DEBUG, $ex) + }; + ($ex:expr) => { + trace_dbg!(level: tracing::Level::DEBUG, $ex) + }; +} + +pub fn version() -> String { + let author = clap::crate_authors!(); + + // let current_exe_path = PathBuf::from(clap::crate_name!()).display().to_string(); + let config_dir_path = get_config_dir().display().to_string(); + let data_dir_path = get_data_dir().display().to_string(); + + format!( + "\ +{VERSION_MESSAGE} + +Authors: {author} + +Config directory: {config_dir_path} +Data directory: {data_dir_path}" + ) +} From 86f6a8c90f5a018b5df9e3974a059853d48e6eb2 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 18 Apr 2024 14:52:05 +0900 Subject: [PATCH 175/205] feat(tui): adding services --- Cargo.lock | 2 + node-tui/.config/config.json5 | 1 + node-tui/Cargo.toml | 6 ++- node-tui/src/action.rs | 8 +--- node-tui/src/app.rs | 2 +- node-tui/src/cli.rs | 6 ++- node-tui/src/components.rs | 2 +- node-tui/src/components/home.rs | 79 ++++++++++++++++++++++++--------- node-tui/src/config.rs | 10 ++--- node-tui/src/main.rs | 5 +-- sn_node_manager/src/cmd/node.rs | 5 +++ sn_peers_acquisition/src/lib.rs | 2 +- 12 files changed, 85 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4ac8ec3783..03b07eabcc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4607,6 +4607,8 @@ dependencies = [ "serde", "serde_json", "signal-hook", + "sn-node-manager", + "sn_peers_acquisition", "sn_service_management", "strip-ansi-escapes", "strum", diff --git a/node-tui/.config/config.json5 b/node-tui/.config/config.json5 index c746239314..dfd8e9bddf 100644 --- a/node-tui/.config/config.json5 +++ b/node-tui/.config/config.json5 @@ -1,6 +1,7 @@ { "keybindings": { "Home": { + "": "StartNodes", // Start nodes "": "Quit", // Quit the application "": "Quit", // Another way to quit "": "Quit", // Yet another way to quit diff --git a/node-tui/Cargo.toml b/node-tui/Cargo.toml index 344996d235..f0025736e5 100644 --- a/node-tui/Cargo.toml +++ b/node-tui/Cargo.toml @@ -10,6 +10,10 @@ build = "build.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +sn_peers_acquisition = { verison = "0.2.10", path = "../sn_peers_acquisition", features = [ + # "network-contacts", +] } +sn_service_management = { verison = "0.2.4", path = "../sn_service_management" } better-panic = "0.3.0" clap = { version = "4.4.5", features = [ "derive", @@ -33,7 +37,7 @@ log = "0.4.20" pretty_assertions = "1.4.0" ratatui = { version = "0.26.0", features = ["serde", "macros"] } serde = { version = "1.0.188", features = ["derive"] } -sn_service_management = { verison = "0.2.4", path = "../sn_service_management" } +sn-node-manager = { verison = "0.7.4", path = "../sn_node_manager" } serde_json = "1.0.107" signal-hook = "0.3.17" strip-ansi-escapes = "0.2.0" diff --git a/node-tui/src/action.rs b/node-tui/src/action.rs index 705e492e6d..95e6b07ed0 100644 --- a/node-tui/src/action.rs +++ b/node-tui/src/action.rs @@ -1,13 +1,9 @@ -use std::{fmt, string::ToString}; - -use serde::{ - de::{self, Deserializer, Visitor}, - Deserialize, Serialize, -}; +use serde::{Deserialize, Serialize}; use strum::Display; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Display, Deserialize)] pub enum Action { + StartNodes, Tick, Render, Resize(u16, u16), diff --git a/node-tui/src/app.rs b/node-tui/src/app.rs index c6c83e0345..812f498f6c 100644 --- a/node-tui/src/app.rs +++ b/node-tui/src/app.rs @@ -1,7 +1,7 @@ use color_eyre::eyre::Result; use crossterm::event::KeyEvent; use ratatui::prelude::Rect; -use serde::{Deserialize, Serialize}; +use sn_peers_acquisition::PeersArgs; use tokio::sync::mpsc; use crate::{ diff --git a/node-tui/src/cli.rs b/node-tui/src/cli.rs index edaaf35b40..d775fad704 100644 --- a/node-tui/src/cli.rs +++ b/node-tui/src/cli.rs @@ -1,6 +1,5 @@ -use std::path::PathBuf; - use clap::Parser; +use sn_peers_acquisition::PeersArgs; use crate::utils::version; @@ -24,4 +23,7 @@ pub struct Cli { default_value_t = 4.0 )] pub frame_rate: f64, + + #[command(flatten)] + pub peers: PeersArgs, } diff --git a/node-tui/src/components.rs b/node-tui/src/components.rs index 6500f3bc5f..6a3ad27375 100644 --- a/node-tui/src/components.rs +++ b/node-tui/src/components.rs @@ -51,7 +51,7 @@ pub trait Component { /// # Returns /// /// * `Result<()>` - An Ok result or an error. - fn init(&mut self, area: Rect) -> Result<()> { + fn init(&mut self, _area: Rect) -> Result<()> { Ok(()) } /// Handle incoming events and produce actions if necessary. diff --git a/node-tui/src/components/home.rs b/node-tui/src/components/home.rs index 1af0c461cb..1ee8147389 100644 --- a/node-tui/src/components/home.rs +++ b/node-tui/src/components/home.rs @@ -1,28 +1,24 @@ -use std::{collections::HashMap, time::Duration}; - use color_eyre::eyre::Result; -use crossterm::event::{KeyCode, KeyEvent}; use ratatui::{prelude::*, widgets::*}; -use serde::{Deserialize, Serialize}; +use sn_peers_acquisition::PeersArgs; use sn_service_management::{get_local_node_registry_path, NodeRegistry, ServiceStatus}; use tokio::sync::mpsc::UnboundedSender; use super::{Component, Frame}; -use crate::{ - action::Action, - config::{Config, KeyBindings}, -}; +use crate::{action::Action, config::Config}; #[derive(Default)] pub struct Home { command_tx: Option>, config: Config, node_registry: Option, + // Network Peers + pub peers_args: PeersArgs, } impl Home { - pub fn new() -> Self { - Self::default() + pub fn new(peers_args: PeersArgs) -> Self { + Self { peers_args, ..Default::default() } } } @@ -39,6 +35,38 @@ impl Component for Home { fn update(&mut self, action: Action) -> Result> { match action { + Action::StartNodes => { + tracing::debug!("STARTING"); + // let local_node_registry = NodeRegistry::load(&get_local_node_registry_path()?)?; + let peers = self.peers_args.clone(); + tokio::spawn(async { + if let Err(err) = sn_node_manager::cmd::node::add( + None, + None, + None, + true, + None, + None, + None, + peers, + None, + None, + None, + None, + None, + None, + sn_node_manager::VerbosityLevel::Minimal, + ) + .await + { + tracing::error!("ERRROR adding {err:?}") + } + + tracing::debug!("added servicionssss"); + + sn_node_manager::cmd::node::start(1, vec![], vec![], sn_node_manager::VerbosityLevel::Minimal) + }); + }, Action::Tick => { let local_node_registry = NodeRegistry::load(&get_local_node_registry_path()?)?; @@ -49,7 +77,7 @@ impl Component for Home { } }, Action::StartNode => { - let local_node_registry = NodeRegistry::load(&get_local_node_registry_path()?)?; + let _local_node_registry = NodeRegistry::load(&get_local_node_registry_path()?)?; }, _ => {}, } @@ -57,6 +85,17 @@ impl Component for Home { } fn draw(&mut self, f: &mut Frame<'_>, area: Rect) -> Result<()> { + // basic home layout + let home_layout = + Layout::new(Direction::Vertical, [Constraint::Min(5), Constraint::Min(3), Constraint::Max(3)]).split(area); + + // top section + f.render_widget( + Paragraph::new("TODO: All Node Stats") + .block(Block::default().title("Autonomi Node Runner").borders(Borders::ALL)), + home_layout[0], + ); + if let Some(registry) = &self.node_registry { let nodes: Vec<_> = registry @@ -73,16 +112,8 @@ impl Component for Home { .collect(); if !nodes.is_empty() { - let mut list = List::new(nodes); - let home_layout = - Layout::new(Direction::Horizontal, [Constraint::Min(5), Constraint::Min(0), Constraint::Length(1)]) - .split(area); + let list = List::new(nodes); - f.render_widget( - Paragraph::new("TODO: All Node Stats") - .block(Block::default().title("Autonomi Node Runner").borders(Borders::ALL)), - home_layout[0], - ); f.render_widget( list.block(Block::default().title("Running nodes").borders(Borders::ALL)), home_layout[1], @@ -92,9 +123,15 @@ impl Component for Home { f.render_widget( Paragraph::new("No nodes running") .block(Block::default().title("Autonomi Node Runner").borders(Borders::ALL)), - area, + home_layout[1], ) } + + f.render_widget( + Paragraph::new("[S]tart nodes, [Q]uit") + .block(Block::default().title(" Key commands ").borders(Borders::ALL)), + home_layout[2], + ); Ok(()) } } diff --git a/node-tui/src/config.rs b/node-tui/src/config.rs index 110533bc06..a9efad6bb2 100644 --- a/node-tui/src/config.rs +++ b/node-tui/src/config.rs @@ -1,15 +1,11 @@ -use std::{collections::HashMap, fmt, path::PathBuf}; +use std::{collections::HashMap, path::PathBuf}; use color_eyre::eyre::Result; -use config::Value; + use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; use derive_deref::{Deref, DerefMut}; use ratatui::style::{Color, Modifier, Style}; -use serde::{ - de::{self, Deserializer, MapAccess, Visitor}, - Deserialize, Serialize, -}; -use serde_json::Value as JsonValue; +use serde::{de::Deserializer, Deserialize}; use crate::{action::Action, mode::Mode}; diff --git a/node-tui/src/main.rs b/node-tui/src/main.rs index 18f66847db..e921ddf2bb 100644 --- a/node-tui/src/main.rs +++ b/node-tui/src/main.rs @@ -1,5 +1,4 @@ /// initially generated from: https://github.com/ratatui-org/templates/blob/main/component/README.md -/// pub mod action; pub mod app; pub mod cli; @@ -15,7 +14,7 @@ use color_eyre::eyre::Result; use crate::{ app::App, - utils::{initialize_logging, initialize_panic_handler, version}, + utils::{initialize_logging, initialize_panic_handler}, }; async fn tokio_main() -> Result<()> { @@ -24,7 +23,7 @@ async fn tokio_main() -> Result<()> { initialize_panic_handler()?; let args = Cli::parse(); - let mut app = App::new(args.tick_rate, args.frame_rate)?; + let mut app = App::new(args.tick_rate, args.frame_rate, args.peers)?; app.run().await?; Ok(()) diff --git a/sn_node_manager/src/cmd/node.rs b/sn_node_manager/src/cmd/node.rs index aa563922ae..155285c2f8 100644 --- a/sn_node_manager/src/cmd/node.rs +++ b/sn_node_manager/src/cmd/node.rs @@ -78,10 +78,14 @@ pub async fn add( let version = get_bin_version(&path)?; (path, version) } else { + tracing::debug!("upsss"); + panic!("uppssssss"); download_and_extract_release(ReleaseType::Safenode, url.clone(), version, &*release_repo) .await? }; + tracing::debug!("peers questions..."); + // Handle the `PeersNotObtained` error to make the `--peer` argument optional for the node // manager. // @@ -124,6 +128,7 @@ pub async fn add( add_node(options, &mut node_registry, &service_manager, verbosity).await?; node_registry.save()?; + tracing::debug!("registery saved......"); Ok(()) } diff --git a/sn_peers_acquisition/src/lib.rs b/sn_peers_acquisition/src/lib.rs index e1698ea374..d68e5e6efd 100644 --- a/sn_peers_acquisition/src/lib.rs +++ b/sn_peers_acquisition/src/lib.rs @@ -37,7 +37,7 @@ const MAX_NETWORK_CONTACTS_GET_RETRIES: usize = 3; /// The name of the environment variable that can be used to pass peers to the node. pub const SAFE_PEERS_ENV: &str = "SAFE_PEERS"; -#[derive(Args, Debug)] +#[derive(Args, Debug, Default, Clone)] pub struct PeersArgs { /// Set to indicate this is the first node in a new network /// From 304bf0b7124cd97dd541db50618dee15e0004103 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Thu, 18 Apr 2024 15:27:32 +0530 Subject: [PATCH 176/205] fix: use tokio localset --- node-tui/src/components/home.rs | 4 ++-- node-tui/src/main.rs | 19 +++++++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/node-tui/src/components/home.rs b/node-tui/src/components/home.rs index 1ee8147389..862df643d3 100644 --- a/node-tui/src/components/home.rs +++ b/node-tui/src/components/home.rs @@ -39,7 +39,7 @@ impl Component for Home { tracing::debug!("STARTING"); // let local_node_registry = NodeRegistry::load(&get_local_node_registry_path()?)?; let peers = self.peers_args.clone(); - tokio::spawn(async { + tokio::task::spawn_local(async { if let Err(err) = sn_node_manager::cmd::node::add( None, None, @@ -64,7 +64,7 @@ impl Component for Home { tracing::debug!("added servicionssss"); - sn_node_manager::cmd::node::start(1, vec![], vec![], sn_node_manager::VerbosityLevel::Minimal) + sn_node_manager::cmd::node::start(1, vec![], vec![], sn_node_manager::VerbosityLevel::Minimal).await }); }, Action::Tick => { diff --git a/node-tui/src/main.rs b/node-tui/src/main.rs index e921ddf2bb..96e9fdff83 100644 --- a/node-tui/src/main.rs +++ b/node-tui/src/main.rs @@ -11,6 +11,7 @@ pub mod utils; use clap::Parser; use cli::Cli; use color_eyre::eyre::Result; +use tokio::{runtime::Runtime, task::LocalSet}; use crate::{ app::App, @@ -31,10 +32,16 @@ async fn tokio_main() -> Result<()> { #[tokio::main] async fn main() -> Result<()> { - if let Err(e) = tokio_main().await { - eprintln!("{} error: Something went wrong", env!("CARGO_PKG_NAME")); - Err(e) - } else { - Ok(()) - } + // Construct a local task set that can run `!Send` futures. + let local = LocalSet::new(); + local + .run_until(async { + if let Err(e) = tokio_main().await { + eprintln!("{} error: Something went wrong", env!("CARGO_PKG_NAME")); + Err(e) + } else { + Ok(()) + } + }) + .await } From 49a145699798811d8afa8ba7026d63a3b634426e Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Fri, 19 Apr 2024 09:00:20 +0900 Subject: [PATCH 177/205] chore: properly use node registry and surface peer ids if they're not removed --- node-tui/.config/config.json5 | 1 + node-tui/Cargo.toml | 2 +- node-tui/src/action.rs | 3 + node-tui/src/app.rs | 3 +- node-tui/src/components/home.rs | 164 +++++++++++++++++++++----------- sn_node_manager/src/cmd/node.rs | 2 - 6 files changed, 118 insertions(+), 57 deletions(-) diff --git a/node-tui/.config/config.json5 b/node-tui/.config/config.json5 index dfd8e9bddf..2ae5768bbf 100644 --- a/node-tui/.config/config.json5 +++ b/node-tui/.config/config.json5 @@ -1,6 +1,7 @@ { "keybindings": { "Home": { + "": "AddNode", // Add nodes "": "StartNodes", // Start nodes "": "Quit", // Quit the application "": "Quit", // Another way to quit diff --git a/node-tui/Cargo.toml b/node-tui/Cargo.toml index f0025736e5..b3f9a22f74 100644 --- a/node-tui/Cargo.toml +++ b/node-tui/Cargo.toml @@ -11,7 +11,7 @@ build = "build.rs" [dependencies] sn_peers_acquisition = { verison = "0.2.10", path = "../sn_peers_acquisition", features = [ - # "network-contacts", + "network-contacts", ] } sn_service_management = { verison = "0.2.4", path = "../sn_service_management" } better-panic = "0.3.0" diff --git a/node-tui/src/action.rs b/node-tui/src/action.rs index 95e6b07ed0..271ff94c13 100644 --- a/node-tui/src/action.rs +++ b/node-tui/src/action.rs @@ -1,8 +1,11 @@ use serde::{Deserialize, Serialize}; +use sn_node_manager::cmd::node::ProgressType; use strum::Display; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Display, Deserialize)] pub enum Action { + ProgressMessage(ProgressType), + AddNode, StartNodes, Tick, Render, diff --git a/node-tui/src/app.rs b/node-tui/src/app.rs index 812f498f6c..94b062f171 100644 --- a/node-tui/src/app.rs +++ b/node-tui/src/app.rs @@ -25,7 +25,8 @@ pub struct App { impl App { pub fn new(tick_rate: f64, frame_rate: f64) -> Result { - let home = Home::new(); + let mut home = Home::new(); + home.check_for_node_registry()?; let fps = FpsCounter::default(); let config = Config::new()?; let mode = Mode::Home; diff --git a/node-tui/src/components/home.rs b/node-tui/src/components/home.rs index 862df643d3..bfa2e3857e 100644 --- a/node-tui/src/components/home.rs +++ b/node-tui/src/components/home.rs @@ -1,8 +1,9 @@ use color_eyre::eyre::Result; use ratatui::{prelude::*, widgets::*}; +use sn_node_manager::{cmd::node::ProgressType, config::get_node_registry_path}; use sn_peers_acquisition::PeersArgs; use sn_service_management::{get_local_node_registry_path, NodeRegistry, ServiceStatus}; -use tokio::sync::mpsc::UnboundedSender; +use tokio::sync::mpsc::{self, UnboundedSender}; use super::{Component, Frame}; use crate::{action::Action, config::Config}; @@ -11,6 +12,7 @@ use crate::{action::Action, config::Config}; pub struct Home { command_tx: Option>, config: Config, + progress_messages: Vec, node_registry: Option, // Network Peers pub peers_args: PeersArgs, @@ -20,6 +22,31 @@ impl Home { pub fn new(peers_args: PeersArgs) -> Self { Self { peers_args, ..Default::default() } } + + // TODO: we should have a helper to correctlt choose the registry + pub fn check_for_node_registry(&mut self) -> Result<()> { + if self.node_registry.is_none() { + tracing::debug!("No registryllocal yet..."); + let reg = NodeRegistry::load(&get_local_node_registry_path()?)?; + + // register this + if !reg.nodes.is_empty() { + self.node_registry = Some(reg); + } + } + + // local nodes failed, so we try the _other_ setup + if self.node_registry.is_none() { + tracing::debug!("No registry yet..."); + let reg = NodeRegistry::load(&get_node_registry_path()?)?; + // register this + if !reg.nodes.is_empty() { + self.node_registry = Some(reg); + } + } + + Ok(()) + } } impl Component for Home { @@ -34,51 +61,77 @@ impl Component for Home { } fn update(&mut self, action: Action) -> Result> { + let action_sender = self.command_tx.clone(); match action { + Action::AddNode => { + tracing::debug!("adding"); + + if let Some(reg) = &self.node_registry { + tracing::debug!("No nodes yet..."); + + if reg.nodes.is_empty() { + let peers = self.peers_args.clone(); + + // report progress via forwarding messages as actions + let (progress_sender, mut progress_receiver) = mpsc::channel::(1); + + tokio::task::spawn_local(async { + if let Err(err) = sn_node_manager::cmd::node::add( + None, + None, + None, + true, + None, + None, + None, + peers, + None, + None, + None, + None, + None, + None, + sn_node_manager::VerbosityLevel::Minimal, + progress_sender, + ) + .await + { + tracing::error!("ERRROR adding {err:?}") + } + }); + + tracing::debug!("OKAY AND NOW LISTENING TO EVENTS>....."); + + let action_sender = self.command_tx.clone(); + tokio::task::spawn(async move { + while let Some(report) = progress_receiver.recv().await { + tracing::debug!("REPORTTTT {report:?}"); + if let Some(ref sender) = action_sender { + if let Err(error) = sender.send(Action::ProgressMessage(report)) { + tracing::error!("Err sending progress action: {error:?}"); + }; + } + } + }); + + tracing::debug!("added servicionssss"); + } else { + tracing::debug!("We've no node registery... so skip the addition of services..."); + } + } + }, Action::StartNodes => { tracing::debug!("STARTING"); - // let local_node_registry = NodeRegistry::load(&get_local_node_registry_path()?)?; - let peers = self.peers_args.clone(); - tokio::task::spawn_local(async { - if let Err(err) = sn_node_manager::cmd::node::add( - None, - None, - None, - true, - None, - None, - None, - peers, - None, - None, - None, - None, - None, - None, - sn_node_manager::VerbosityLevel::Minimal, - ) - .await - { - tracing::error!("ERRROR adding {err:?}") - } - - tracing::debug!("added servicionssss"); + tokio::task::spawn_local(async { sn_node_manager::cmd::node::start(1, vec![], vec![], sn_node_manager::VerbosityLevel::Minimal).await }); }, + Action::ProgressMessage(message) => self.progress_messages.push(message), Action::Tick => { - let local_node_registry = NodeRegistry::load(&get_local_node_registry_path()?)?; - - if !local_node_registry.nodes.is_empty() { - self.node_registry = Some(local_node_registry); - } else { - self.node_registry = None; - } - }, - Action::StartNode => { - let _local_node_registry = NodeRegistry::load(&get_local_node_registry_path()?)?; + self.check_for_node_registry()?; }, + _ => {}, } Ok(None) @@ -90,26 +143,31 @@ impl Component for Home { Layout::new(Direction::Vertical, [Constraint::Min(5), Constraint::Min(3), Constraint::Max(3)]).split(area); // top section + // + // let text = Text::raw(content) + // + let text: String = self.progress_messages.iter().map(|progress| format!("{progress:?}\n")).collect(); f.render_widget( - Paragraph::new("TODO: All Node Stats") - .block(Block::default().title("Autonomi Node Runner").borders(Borders::ALL)), + Paragraph::new(text).block(Block::default().title("Autonomi Node Status").borders(Borders::ALL)), home_layout[0], ); if let Some(registry) = &self.node_registry { - let nodes: Vec<_> = - registry - .to_status_summary() - .nodes - .iter() - .filter_map(|n| { - if let ServiceStatus::Running = n.status { - n.peer_id.map(|p| p.to_string()) - } else { - None - } - }) - .collect(); + let nodes: Vec<_> = registry + .to_status_summary() + .nodes + .iter() + .filter_map(|n| { + let peer_id = n.peer_id; + tracing::debug!("peer_id {:?} {:?}", peer_id, n.status); + if n.status == ServiceStatus::Removed { + return None; + } + + let id = peer_id.map(|p| p.to_string()).unwrap_or("Pending...".to_string()); + Some(format!("{id:?}: {:?}", n.status)) + }) + .collect(); if !nodes.is_empty() { let list = List::new(nodes); @@ -128,7 +186,7 @@ impl Component for Home { } f.render_widget( - Paragraph::new("[S]tart nodes, [Q]uit") + Paragraph::new("[A]dd node, [S]tart node, [Q]uit") .block(Block::default().title(" Key commands ").borders(Borders::ALL)), home_layout[2], ); diff --git a/sn_node_manager/src/cmd/node.rs b/sn_node_manager/src/cmd/node.rs index 155285c2f8..9df0fa4cea 100644 --- a/sn_node_manager/src/cmd/node.rs +++ b/sn_node_manager/src/cmd/node.rs @@ -78,8 +78,6 @@ pub async fn add( let version = get_bin_version(&path)?; (path, version) } else { - tracing::debug!("upsss"); - panic!("uppssssss"); download_and_extract_release(ReleaseType::Safenode, url.clone(), version, &*release_repo) .await? }; From 7bcd148487189555e90525be41aa489824842b07 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Mon, 29 Apr 2024 20:42:49 +0530 Subject: [PATCH 178/205] refactor: focus on remote networks and single threading --- node-tui/src/app.rs | 3 +- node-tui/src/components/home.rs | 124 ++++++++++---------------------- 2 files changed, 37 insertions(+), 90 deletions(-) diff --git a/node-tui/src/app.rs b/node-tui/src/app.rs index 94b062f171..7ec0d7dc37 100644 --- a/node-tui/src/app.rs +++ b/node-tui/src/app.rs @@ -25,8 +25,7 @@ pub struct App { impl App { pub fn new(tick_rate: f64, frame_rate: f64) -> Result { - let mut home = Home::new(); - home.check_for_node_registry()?; + let home = Home::new()?; let fps = FpsCounter::default(); let config = Config::new()?; let mode = Mode::Home; diff --git a/node-tui/src/components/home.rs b/node-tui/src/components/home.rs index bfa2e3857e..fc65f50240 100644 --- a/node-tui/src/components/home.rs +++ b/node-tui/src/components/home.rs @@ -2,7 +2,7 @@ use color_eyre::eyre::Result; use ratatui::{prelude::*, widgets::*}; use sn_node_manager::{cmd::node::ProgressType, config::get_node_registry_path}; use sn_peers_acquisition::PeersArgs; -use sn_service_management::{get_local_node_registry_path, NodeRegistry, ServiceStatus}; +use sn_service_management::{NodeRegistry, ServiceStatus}; use tokio::sync::mpsc::{self, UnboundedSender}; use super::{Component, Frame}; @@ -12,40 +12,20 @@ use crate::{action::Action, config::Config}; pub struct Home { command_tx: Option>, config: Config, - progress_messages: Vec, + // state node_registry: Option, + // Network Peers pub peers_args: PeersArgs, } impl Home { - pub fn new(peers_args: PeersArgs) -> Self { - Self { peers_args, ..Default::default() } - } - - // TODO: we should have a helper to correctlt choose the registry - pub fn check_for_node_registry(&mut self) -> Result<()> { - if self.node_registry.is_none() { - tracing::debug!("No registryllocal yet..."); - let reg = NodeRegistry::load(&get_local_node_registry_path()?)?; - - // register this - if !reg.nodes.is_empty() { - self.node_registry = Some(reg); - } - } + pub fn new(peers_args: PeersArgs) -> Result { + tracing::debug!("Loading node registry"); - // local nodes failed, so we try the _other_ setup - if self.node_registry.is_none() { - tracing::debug!("No registry yet..."); - let reg = NodeRegistry::load(&get_node_registry_path()?)?; - // register this - if !reg.nodes.is_empty() { - self.node_registry = Some(reg); - } - } + let node_registry = NodeRegistry::load(&get_node_registry_path()?)?; - Ok(()) + Ok(Self { peers_args, node_registry: Some(node_registry), ..Default::default() }) } } @@ -61,64 +41,39 @@ impl Component for Home { } fn update(&mut self, action: Action) -> Result> { - let action_sender = self.command_tx.clone(); match action { Action::AddNode => { tracing::debug!("adding"); - if let Some(reg) = &self.node_registry { - tracing::debug!("No nodes yet..."); - - if reg.nodes.is_empty() { - let peers = self.peers_args.clone(); - - // report progress via forwarding messages as actions - let (progress_sender, mut progress_receiver) = mpsc::channel::(1); - - tokio::task::spawn_local(async { - if let Err(err) = sn_node_manager::cmd::node::add( - None, - None, - None, - true, - None, - None, - None, - peers, - None, - None, - None, - None, - None, - None, - sn_node_manager::VerbosityLevel::Minimal, - progress_sender, - ) - .await - { - tracing::error!("ERRROR adding {err:?}") - } - }); - - tracing::debug!("OKAY AND NOW LISTENING TO EVENTS>....."); - - let action_sender = self.command_tx.clone(); - tokio::task::spawn(async move { - while let Some(report) = progress_receiver.recv().await { - tracing::debug!("REPORTTTT {report:?}"); - if let Some(ref sender) = action_sender { - if let Err(error) = sender.send(Action::ProgressMessage(report)) { - tracing::error!("Err sending progress action: {error:?}"); - }; - } - } - }); - - tracing::debug!("added servicionssss"); - } else { - tracing::debug!("We've no node registery... so skip the addition of services..."); + let peers = self.peers_args.clone(); + let (progress_sender, _) = mpsc::channel::(1); + + tokio::task::spawn_local(async { + if let Err(err) = sn_node_manager::cmd::node::add( + None, + None, + None, + true, + None, + None, + None, + peers, + None, + None, + None, + None, + None, + None, + sn_node_manager::VerbosityLevel::Minimal, + progress_sender, + ) + .await + { + tracing::error!("Error while adding service {err:?}") } - } + }); + + tracing::debug!("added servicionssss"); }, Action::StartNodes => { tracing::debug!("STARTING"); @@ -127,10 +82,6 @@ impl Component for Home { sn_node_manager::cmd::node::start(1, vec![], vec![], sn_node_manager::VerbosityLevel::Minimal).await }); }, - Action::ProgressMessage(message) => self.progress_messages.push(message), - Action::Tick => { - self.check_for_node_registry()?; - }, _ => {}, } @@ -144,11 +95,8 @@ impl Component for Home { // top section // - // let text = Text::raw(content) - // - let text: String = self.progress_messages.iter().map(|progress| format!("{progress:?}\n")).collect(); f.render_widget( - Paragraph::new(text).block(Block::default().title("Autonomi Node Status").borders(Borders::ALL)), + Paragraph::new("None").block(Block::default().title("Autonomi Node Status").borders(Borders::ALL)), home_layout[0], ); From 7a115cc6d9bd0e4307ed5ba6a634d4c6d9c1d108 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Mon, 29 Apr 2024 21:36:30 +0530 Subject: [PATCH 179/205] feat: add and start nodes sequentially --- node-tui/src/action.rs | 2 + node-tui/src/components/home.rs | 67 +++++++++++++++++++++++++-------- node-tui/src/main.rs | 3 ++ 3 files changed, 57 insertions(+), 15 deletions(-) diff --git a/node-tui/src/action.rs b/node-tui/src/action.rs index 271ff94c13..427f2be226 100644 --- a/node-tui/src/action.rs +++ b/node-tui/src/action.rs @@ -6,7 +6,9 @@ use strum::Display; pub enum Action { ProgressMessage(ProgressType), AddNode, + AddNodeCompleted, StartNodes, + StartNodesCompleted, Tick, Render, Resize(u16, u16), diff --git a/node-tui/src/components/home.rs b/node-tui/src/components/home.rs index fc65f50240..28237c15c7 100644 --- a/node-tui/src/components/home.rs +++ b/node-tui/src/components/home.rs @@ -1,4 +1,4 @@ -use color_eyre::eyre::Result; +use color_eyre::eyre::{OptionExt, Result}; use ratatui::{prelude::*, widgets::*}; use sn_node_manager::{cmd::node::ProgressType, config::get_node_registry_path}; use sn_peers_acquisition::PeersArgs; @@ -10,10 +10,13 @@ use crate::{action::Action, config::Config}; #[derive(Default)] pub struct Home { - command_tx: Option>, + action_sender: Option>, config: Config, // state node_registry: Option, + // Currently the node registry file does not support concurrent actions and thus can lead to + // inconsistent state. A simple file lock or a db like file would work. + lock_registry: bool, // Network Peers pub peers_args: PeersArgs, @@ -21,7 +24,7 @@ pub struct Home { impl Home { pub fn new(peers_args: PeersArgs) -> Result { - tracing::debug!("Loading node registry"); + debug!("Loading node registry"); let node_registry = NodeRegistry::load(&get_node_registry_path()?)?; @@ -31,7 +34,7 @@ impl Home { impl Component for Home { fn register_action_handler(&mut self, tx: UnboundedSender) -> Result<()> { - self.command_tx = Some(tx); + self.action_sender = Some(tx); Ok(()) } @@ -43,12 +46,18 @@ impl Component for Home { fn update(&mut self, action: Action) -> Result> { match action { Action::AddNode => { - tracing::debug!("adding"); + if self.lock_registry { + error!("Registry is locked, cannot add node now."); + return Ok(None); + } + info!("Adding a new node service"); let peers = self.peers_args.clone(); let (progress_sender, _) = mpsc::channel::(1); + let action_sender = self.get_actions_sender()?; + self.lock_registry = true; - tokio::task::spawn_local(async { + tokio::task::spawn_local(async move { if let Err(err) = sn_node_manager::cmd::node::add( None, None, @@ -69,20 +78,42 @@ impl Component for Home { ) .await { - tracing::error!("Error while adding service {err:?}") + error!("Error while adding service {err:?}") + } + info!("Successfully added service"); + // todo: need to handle these properly? + if let Err(err) = action_sender.send(Action::AddNodeCompleted) { + error!("Error while sending action: {err:?}"); } }); - - tracing::debug!("added servicionssss"); }, Action::StartNodes => { - tracing::debug!("STARTING"); - - tokio::task::spawn_local(async { - sn_node_manager::cmd::node::start(1, vec![], vec![], sn_node_manager::VerbosityLevel::Minimal).await + if self.lock_registry { + error!("Registry is locked. Cannot start node now."); + return Ok(None); + } + info!("Starting Node service"); + let action_sender = self.get_actions_sender()?; + + self.lock_registry = true; + tokio::task::spawn_local(async move { + if let Err(err) = + sn_node_manager::cmd::node::start(1, vec![], vec![], sn_node_manager::VerbosityLevel::Minimal) + .await + { + error!("Error while starting services {err:?}"); + } + if let Err(err) = action_sender.send(Action::StartNodesCompleted) { + error!("Error while sending action: {err:?}"); + } + info!("Successfully started services"); }); }, - + Action::AddNodeCompleted | Action::StartNodesCompleted => { + self.lock_registry = false; + let node_registry = NodeRegistry::load(&get_node_registry_path()?)?; + self.node_registry = Some(node_registry); + }, _ => {}, } Ok(None) @@ -107,7 +138,7 @@ impl Component for Home { .iter() .filter_map(|n| { let peer_id = n.peer_id; - tracing::debug!("peer_id {:?} {:?}", peer_id, n.status); + debug!("peer_id {:?} {:?}", peer_id, n.status); if n.status == ServiceStatus::Removed { return None; } @@ -141,3 +172,9 @@ impl Component for Home { Ok(()) } } + +impl Home { + fn get_actions_sender(&self) -> Result> { + self.action_sender.clone().ok_or_eyre("Action sender not registered") + } +} diff --git a/node-tui/src/main.rs b/node-tui/src/main.rs index 96e9fdff83..830cb79de1 100644 --- a/node-tui/src/main.rs +++ b/node-tui/src/main.rs @@ -8,6 +8,9 @@ pub mod mode; pub mod tui; pub mod utils; +#[macro_use] +extern crate tracing; + use clap::Parser; use cli::Cli; use color_eyre::eyre::Result; From f843510fd0650523c3631b1ab1b4b55796b770bc Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Mon, 29 Apr 2024 22:20:39 +0530 Subject: [PATCH 180/205] feat: enable stopping of node --- node-tui/.config/config.json5 | 5 +++-- node-tui/src/action.rs | 17 +++++++++------ node-tui/src/components/home.rs | 38 +++++++++++++++++++++++++++------ node-tui/src/main.rs | 2 +- 4 files changed, 47 insertions(+), 15 deletions(-) diff --git a/node-tui/.config/config.json5 b/node-tui/.config/config.json5 index 2ae5768bbf..1545f44af7 100644 --- a/node-tui/.config/config.json5 +++ b/node-tui/.config/config.json5 @@ -1,8 +1,9 @@ { "keybindings": { "Home": { - "": "AddNode", // Add nodes - "": "StartNodes", // Start nodes + "": {"HomeActions":"AddNode"}, // Add nodes + "": {"HomeActions":"StartNodes"}, // Start nodes + "": {"HomeActions":"StopNode"}, // Stop nodes "": "Quit", // Quit the application "": "Quit", // Another way to quit "": "Quit", // Yet another way to quit diff --git a/node-tui/src/action.rs b/node-tui/src/action.rs index 427f2be226..1ec2bb59e9 100644 --- a/node-tui/src/action.rs +++ b/node-tui/src/action.rs @@ -1,14 +1,9 @@ use serde::{Deserialize, Serialize}; -use sn_node_manager::cmd::node::ProgressType; use strum::Display; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Display, Deserialize)] pub enum Action { - ProgressMessage(ProgressType), - AddNode, - AddNodeCompleted, - StartNodes, - StartNodesCompleted, + HomeActions(HomeActions), Tick, Render, Resize(u16, u16), @@ -20,3 +15,13 @@ pub enum Action { Error(String), Help, } + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Display, Deserialize)] +pub enum HomeActions { + AddNode, + AddNodeCompleted, + StartNodes, + StartNodesCompleted, + StopNode, + StopNodeCompleted, +} diff --git a/node-tui/src/components/home.rs b/node-tui/src/components/home.rs index 28237c15c7..6937a09349 100644 --- a/node-tui/src/components/home.rs +++ b/node-tui/src/components/home.rs @@ -6,7 +6,10 @@ use sn_service_management::{NodeRegistry, ServiceStatus}; use tokio::sync::mpsc::{self, UnboundedSender}; use super::{Component, Frame}; -use crate::{action::Action, config::Config}; +use crate::{ + action::{Action, HomeActions}, + config::Config, +}; #[derive(Default)] pub struct Home { @@ -45,7 +48,7 @@ impl Component for Home { fn update(&mut self, action: Action) -> Result> { match action { - Action::AddNode => { + Action::HomeActions(HomeActions::AddNode) => { if self.lock_registry { error!("Registry is locked, cannot add node now."); return Ok(None); @@ -82,12 +85,12 @@ impl Component for Home { } info!("Successfully added service"); // todo: need to handle these properly? - if let Err(err) = action_sender.send(Action::AddNodeCompleted) { + if let Err(err) = action_sender.send(Action::HomeActions(HomeActions::AddNodeCompleted)) { error!("Error while sending action: {err:?}"); } }); }, - Action::StartNodes => { + Action::HomeActions(HomeActions::StartNodes) => { if self.lock_registry { error!("Registry is locked. Cannot start node now."); return Ok(None); @@ -103,13 +106,36 @@ impl Component for Home { { error!("Error while starting services {err:?}"); } - if let Err(err) = action_sender.send(Action::StartNodesCompleted) { + if let Err(err) = action_sender.send(Action::HomeActions(HomeActions::StartNodesCompleted)) { error!("Error while sending action: {err:?}"); } info!("Successfully started services"); }); }, - Action::AddNodeCompleted | Action::StartNodesCompleted => { + Action::HomeActions(HomeActions::StopNode) => { + if self.lock_registry { + error!("Registry is locked. Cannot stop node now."); + return Ok(None); + } + info!("Stopping node service"); + let action_sender = self.get_actions_sender()?; + + self.lock_registry = true; + tokio::task::spawn_local(async move { + if let Err(err) = + sn_node_manager::cmd::node::stop(vec![], vec![], sn_node_manager::VerbosityLevel::Minimal).await + { + error!("Error while stopping services {err:?}"); + } + if let Err(err) = action_sender.send(Action::HomeActions(HomeActions::StopNodeCompleted)) { + error!("Error while sending action: {err:?}"); + } + info!("Successfully stopped services"); + }); + }, + Action::HomeActions(HomeActions::AddNodeCompleted) + | Action::HomeActions(HomeActions::StartNodesCompleted) + | Action::HomeActions(HomeActions::StopNodeCompleted) => { self.lock_registry = false; let node_registry = NodeRegistry::load(&get_node_registry_path()?)?; self.node_registry = Some(node_registry); diff --git a/node-tui/src/main.rs b/node-tui/src/main.rs index 830cb79de1..521e045ad6 100644 --- a/node-tui/src/main.rs +++ b/node-tui/src/main.rs @@ -14,7 +14,7 @@ extern crate tracing; use clap::Parser; use cli::Cli; use color_eyre::eyre::Result; -use tokio::{runtime::Runtime, task::LocalSet}; +use tokio::task::LocalSet; use crate::{ app::App, From 4b29854be76ee5f4f1d8198c8159fc79e3af3b5e Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 30 Apr 2024 20:54:32 +0530 Subject: [PATCH 181/205] feat(tui): display nodes in a stateful table --- node-tui/src/components/home.rs | 117 +++++++++++++++++++++----------- 1 file changed, 76 insertions(+), 41 deletions(-) diff --git a/node-tui/src/components/home.rs b/node-tui/src/components/home.rs index 6937a09349..4dc4334630 100644 --- a/node-tui/src/components/home.rs +++ b/node-tui/src/components/home.rs @@ -2,7 +2,7 @@ use color_eyre::eyre::{OptionExt, Result}; use ratatui::{prelude::*, widgets::*}; use sn_node_manager::{cmd::node::ProgressType, config::get_node_registry_path}; use sn_peers_acquisition::PeersArgs; -use sn_service_management::{NodeRegistry, ServiceStatus}; +use sn_service_management::{NodeRegistry, NodeServiceData, ServiceStatus}; use tokio::sync::mpsc::{self, UnboundedSender}; use super::{Component, Frame}; @@ -16,7 +16,8 @@ pub struct Home { action_sender: Option>, config: Config, // state - node_registry: Option, + running_nodes: Vec, + node_table_state: TableState, // Currently the node registry file does not support concurrent actions and thus can lead to // inconsistent state. A simple file lock or a db like file would work. lock_registry: bool, @@ -27,11 +28,9 @@ pub struct Home { impl Home { pub fn new(peers_args: PeersArgs) -> Result { - debug!("Loading node registry"); - - let node_registry = NodeRegistry::load(&get_node_registry_path()?)?; - - Ok(Self { peers_args, node_registry: Some(node_registry), ..Default::default() }) + let mut home = Self { peers_args, ..Default::default() }; + home.load_node_registry()?; + Ok(home) } } @@ -137,8 +136,7 @@ impl Component for Home { | Action::HomeActions(HomeActions::StartNodesCompleted) | Action::HomeActions(HomeActions::StopNodeCompleted) => { self.lock_registry = false; - let node_registry = NodeRegistry::load(&get_node_registry_path()?)?; - self.node_registry = Some(node_registry); + self.load_node_registry()?; }, _ => {}, } @@ -157,41 +155,37 @@ impl Component for Home { home_layout[0], ); - if let Some(registry) = &self.node_registry { - let nodes: Vec<_> = registry - .to_status_summary() - .nodes - .iter() - .filter_map(|n| { - let peer_id = n.peer_id; - debug!("peer_id {:?} {:?}", peer_id, n.status); - if n.status == ServiceStatus::Removed { - return None; - } + // Node List + let rows: Vec<_> = self + .running_nodes + .iter() + .filter_map(|n| { + let peer_id = n.peer_id; + info!("peer_id {:?} {:?}", peer_id, n.status); + if n.status == ServiceStatus::Removed { + return None; + } + let service_name = n.service_name.clone(); + let peer_id = peer_id.map(|p| p.to_string()).unwrap_or("-".to_string()); + let status = format!("{:?}", n.status); - let id = peer_id.map(|p| p.to_string()).unwrap_or("Pending...".to_string()); - Some(format!("{id:?}: {:?}", n.status)) - }) - .collect(); - - if !nodes.is_empty() { - let list = List::new(nodes); - - f.render_widget( - list.block(Block::default().title("Running nodes").borders(Borders::ALL)), - home_layout[1], - ); - } - } else { - f.render_widget( - Paragraph::new("No nodes running") - .block(Block::default().title("Autonomi Node Runner").borders(Borders::ALL)), - home_layout[1], - ) - } + let row = vec![service_name, peer_id, status]; + Some(Row::new(row)) + }) + .collect(); + + let widths = [Constraint::Max(15), Constraint::Min(30), Constraint::Max(10)]; + let table = Table::new(rows, widths) + .column_spacing(2) + .header(Row::new(vec!["Service", "PeerId", "Status"]).style(Style::new().bold()).bottom_margin(1)) + .highlight_style(Style::new().reversed()) + .block(Block::default().title("Running Nodes").borders(Borders::ALL)) + .highlight_symbol(">"); + + f.render_stateful_widget(table, home_layout[1], &mut self.node_table_state); f.render_widget( - Paragraph::new("[A]dd node, [S]tart node, [Q]uit") + Paragraph::new("[A]dd node, [S]tart node, [K]ill node, [Q]uit") .block(Block::default().title(" Key commands ").borders(Borders::ALL)), home_layout[2], ); @@ -203,4 +197,45 @@ impl Home { fn get_actions_sender(&self) -> Result> { self.action_sender.clone().ok_or_eyre("Action sender not registered") } + + fn load_node_registry(&mut self) -> Result<()> { + let node_registry = NodeRegistry::load(&get_node_registry_path()?)?; + self.running_nodes = + node_registry.nodes.into_iter().filter(|node| node.status != ServiceStatus::Removed).collect(); + info!("Loaded node registry. Runnign nodes: {:?}", self.running_nodes.len()); + + Ok(()) + } + + fn next_item_in_table(&mut self) { + let i = match self.node_table_state.selected() { + Some(i) => { + if i >= self.running_nodes.len() - 1 { + 0 + } else { + i + 1 + } + }, + None => 0, + }; + self.node_table_state.select(Some(i)); + } + + fn previous_item_in_table(&mut self) { + let i = match self.node_table_state.selected() { + Some(i) => { + if i == 0 { + self.running_nodes.len() - 1 + } else { + i - 1 + } + }, + None => 0, + }; + self.node_table_state.select(Some(i)); + } + + fn unselect_table_item(&mut self) { + self.node_table_state.select(None); + } } From 479bfe5be04db0aebf3f1e0b8bcc9e2d60a54909 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Tue, 30 Apr 2024 21:11:40 +0530 Subject: [PATCH 182/205] feat(tui): select different items from the table --- node-tui/.config/config.json5 | 2 ++ node-tui/src/action.rs | 3 +++ node-tui/src/cli.rs | 2 +- node-tui/src/components/home.rs | 10 ++++++++-- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/node-tui/.config/config.json5 b/node-tui/.config/config.json5 index 1545f44af7..7178fea801 100644 --- a/node-tui/.config/config.json5 +++ b/node-tui/.config/config.json5 @@ -4,6 +4,8 @@ "": {"HomeActions":"AddNode"}, // Add nodes "": {"HomeActions":"StartNodes"}, // Start nodes "": {"HomeActions":"StopNode"}, // Stop nodes + "up" : {"HomeActions":"PreviousTableItem"}, + "down" : {"HomeActions":"NextTableItem"}, "": "Quit", // Quit the application "": "Quit", // Another way to quit "": "Quit", // Yet another way to quit diff --git a/node-tui/src/action.rs b/node-tui/src/action.rs index 1ec2bb59e9..6de4aac602 100644 --- a/node-tui/src/action.rs +++ b/node-tui/src/action.rs @@ -24,4 +24,7 @@ pub enum HomeActions { StartNodesCompleted, StopNode, StopNodeCompleted, + + PreviousTableItem, + NextTableItem, } diff --git a/node-tui/src/cli.rs b/node-tui/src/cli.rs index d775fad704..b0ac6f372c 100644 --- a/node-tui/src/cli.rs +++ b/node-tui/src/cli.rs @@ -20,7 +20,7 @@ pub struct Cli { long, value_name = "FLOAT", help = "Frame rate, i.e. number of frames per second", - default_value_t = 4.0 + default_value_t = 60.0 )] pub frame_rate: f64, diff --git a/node-tui/src/components/home.rs b/node-tui/src/components/home.rs index 4dc4334630..0cf46d3030 100644 --- a/node-tui/src/components/home.rs +++ b/node-tui/src/components/home.rs @@ -138,6 +138,12 @@ impl Component for Home { self.lock_registry = false; self.load_node_registry()?; }, + Action::HomeActions(HomeActions::PreviousTableItem) => { + self.select_previous_table_item(); + }, + Action::HomeActions(HomeActions::NextTableItem) => { + self.select_next_table_item(); + }, _ => {}, } Ok(None) @@ -207,7 +213,7 @@ impl Home { Ok(()) } - fn next_item_in_table(&mut self) { + fn select_next_table_item(&mut self) { let i = match self.node_table_state.selected() { Some(i) => { if i >= self.running_nodes.len() - 1 { @@ -221,7 +227,7 @@ impl Home { self.node_table_state.select(Some(i)); } - fn previous_item_in_table(&mut self) { + fn select_previous_table_item(&mut self) { let i = match self.node_table_state.selected() { Some(i) => { if i == 0 { From 753570faa742b8439f6d3d350ca3fb3670e410d4 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 1 May 2024 14:51:48 +0530 Subject: [PATCH 183/205] refactor(tui): remove fps component --- node-tui/src/app.rs | 9 ++-- node-tui/src/components.rs | 1 - node-tui/src/components/fps.rs | 90 ---------------------------------- node-tui/src/config.rs | 7 ++- 4 files changed, 7 insertions(+), 100 deletions(-) delete mode 100644 node-tui/src/components/fps.rs diff --git a/node-tui/src/app.rs b/node-tui/src/app.rs index 7ec0d7dc37..d9edf51ed8 100644 --- a/node-tui/src/app.rs +++ b/node-tui/src/app.rs @@ -6,7 +6,7 @@ use tokio::sync::mpsc; use crate::{ action::Action, - components::{fps::FpsCounter, home::Home, Component}, + components::{home::Home, Component}, config::Config, mode::Mode, tui, @@ -24,15 +24,14 @@ pub struct App { } impl App { - pub fn new(tick_rate: f64, frame_rate: f64) -> Result { - let home = Home::new()?; - let fps = FpsCounter::default(); + pub fn new(tick_rate: f64, frame_rate: f64, peers_args: PeersArgs) -> Result { + let home = Home::new(peers_args)?; let config = Config::new()?; let mode = Mode::Home; Ok(Self { tick_rate, frame_rate, - components: vec![Box::new(home), Box::new(fps)], + components: vec![Box::new(home)], should_quit: false, should_suspend: false, config, diff --git a/node-tui/src/components.rs b/node-tui/src/components.rs index 6a3ad27375..4c643a663b 100644 --- a/node-tui/src/components.rs +++ b/node-tui/src/components.rs @@ -9,7 +9,6 @@ use crate::{ tui::{Event, Frame}, }; -pub mod fps; pub mod home; /// `Component` is a trait that represents a visual and interactive element of the user interface. diff --git a/node-tui/src/components/fps.rs b/node-tui/src/components/fps.rs deleted file mode 100644 index a775bc483f..0000000000 --- a/node-tui/src/components/fps.rs +++ /dev/null @@ -1,90 +0,0 @@ -use std::time::Instant; - -use color_eyre::eyre::Result; -use ratatui::{prelude::*, widgets::*}; - -use super::Component; -use crate::{action::Action, tui::Frame}; - -#[derive(Debug, Clone, PartialEq)] -pub struct FpsCounter { - app_start_time: Instant, - app_frames: u32, - app_fps: f64, - - render_start_time: Instant, - render_frames: u32, - render_fps: f64, -} - -impl Default for FpsCounter { - fn default() -> Self { - Self::new() - } -} - -impl FpsCounter { - pub fn new() -> Self { - Self { - app_start_time: Instant::now(), - app_frames: 0, - app_fps: 0.0, - render_start_time: Instant::now(), - render_frames: 0, - render_fps: 0.0, - } - } - - fn app_tick(&mut self) -> Result<()> { - self.app_frames += 1; - let now = Instant::now(); - let elapsed = (now - self.app_start_time).as_secs_f64(); - if elapsed >= 1.0 { - self.app_fps = self.app_frames as f64 / elapsed; - self.app_start_time = now; - self.app_frames = 0; - } - Ok(()) - } - - fn render_tick(&mut self) -> Result<()> { - self.render_frames += 1; - let now = Instant::now(); - let elapsed = (now - self.render_start_time).as_secs_f64(); - if elapsed >= 1.0 { - self.render_fps = self.render_frames as f64 / elapsed; - self.render_start_time = now; - self.render_frames = 0; - } - Ok(()) - } -} - -impl Component for FpsCounter { - fn update(&mut self, action: Action) -> Result> { - if let Action::Tick = action { - self.app_tick()? - }; - if let Action::Render = action { - self.render_tick()? - }; - Ok(None) - } - - fn draw(&mut self, f: &mut Frame<'_>, rect: Rect) -> Result<()> { - let rects = Layout::default() - .direction(Direction::Vertical) - .constraints(vec![ - Constraint::Length(1), // first row - Constraint::Min(0), - ]) - .split(rect); - - let rect = rects[0]; - - let s = format!("{:.2} ticks per sec (app) {:.2} frames per sec (render)", self.app_fps, self.render_fps); - let block = Block::default().title(block::Title::from(s.dim()).alignment(Alignment::Right)); - f.render_widget(block, rect); - Ok(()) - } -} diff --git a/node-tui/src/config.rs b/node-tui/src/config.rs index a9efad6bb2..359bfa58e4 100644 --- a/node-tui/src/config.rs +++ b/node-tui/src/config.rs @@ -1,7 +1,6 @@ use std::{collections::HashMap, path::PathBuf}; use color_eyre::eyre::Result; - use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; use derive_deref::{Deref, DerefMut}; use ratatui::style::{Color, Modifier, Style}; @@ -67,7 +66,7 @@ impl Config { for (mode, default_styles) in default_config.styles.iter() { let user_styles = cfg.styles.entry(*mode).or_default(); for (style_key, style) in default_styles.iter() { - user_styles.entry(style_key.clone()).or_insert_with(|| style.clone()); + user_styles.entry(style_key.clone()).or_insert_with(|| *style); } } @@ -197,7 +196,7 @@ pub fn key_event_to_string(key_event: &KeyEvent) -> String { char = format!("f({c})"); &char }, - KeyCode::Char(c) if c == ' ' => "space", + KeyCode::Char(' ') => "space", KeyCode::Char(c) => { char = c.to_string(); &char @@ -426,7 +425,7 @@ mod tests { #[test] fn test_parse_color_rgb() { let color = parse_color("rgb123"); - let expected = 16 + 1 * 36 + 2 * 6 + 3; + let expected = 16 + 36 + 2 * 6 + 3; assert_eq!(color, Some(Color::Indexed(expected))); } From 1db19e12b471ed8151fa25bc7950f78dd155003b Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 1 May 2024 15:01:49 +0530 Subject: [PATCH 184/205] chore(tui): add copyright message --- node-tui/Cargo.toml | 1 - node-tui/README.md | 2 -- node-tui/build.rs | 8 ++++++++ node-tui/rust-toolchain.toml | 2 -- node-tui/src/action.rs | 8 ++++++++ node-tui/src/app.rs | 8 ++++++++ node-tui/src/cli.rs | 8 ++++++++ node-tui/src/components.rs | 8 ++++++++ node-tui/src/components/home.rs | 8 ++++++++ node-tui/src/config.rs | 8 ++++++++ node-tui/src/main.rs | 9 ++++++++- node-tui/src/mode.rs | 8 ++++++++ node-tui/src/tui.rs | 8 ++++++++ node-tui/src/utils.rs | 8 ++++++++ 14 files changed, 88 insertions(+), 6 deletions(-) delete mode 100644 node-tui/rust-toolchain.toml diff --git a/node-tui/Cargo.toml b/node-tui/Cargo.toml index b3f9a22f74..a313cc989f 100644 --- a/node-tui/Cargo.toml +++ b/node-tui/Cargo.toml @@ -7,7 +7,6 @@ description = "Terminal interface for autonomi node management" authors = ["Josh Wilson "] build = "build.rs" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] sn_peers_acquisition = { verison = "0.2.10", path = "../sn_peers_acquisition", features = [ diff --git a/node-tui/README.md b/node-tui/README.md index 9fcf95277b..73c34ec0a5 100644 --- a/node-tui/README.md +++ b/node-tui/README.md @@ -1,5 +1,3 @@ # node-tui -[![CI](https://github.com//node-tui/workflows/CI/badge.svg)](https://github.com//node-tui/actions) - Terminal interface for autonomi node management diff --git a/node-tui/build.rs b/node-tui/build.rs index cb965a14a1..23757b5cbd 100644 --- a/node-tui/build.rs +++ b/node-tui/build.rs @@ -1,3 +1,11 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + fn main() -> Result<(), Box> { vergen::EmitBuilder::builder().all_build().all_git().emit()?; Ok(()) diff --git a/node-tui/rust-toolchain.toml b/node-tui/rust-toolchain.toml deleted file mode 100644 index 5d56faf9ae..0000000000 --- a/node-tui/rust-toolchain.toml +++ /dev/null @@ -1,2 +0,0 @@ -[toolchain] -channel = "nightly" diff --git a/node-tui/src/action.rs b/node-tui/src/action.rs index 6de4aac602..99c187404f 100644 --- a/node-tui/src/action.rs +++ b/node-tui/src/action.rs @@ -1,3 +1,11 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + use serde::{Deserialize, Serialize}; use strum::Display; diff --git a/node-tui/src/app.rs b/node-tui/src/app.rs index d9edf51ed8..3e1d3ea2ea 100644 --- a/node-tui/src/app.rs +++ b/node-tui/src/app.rs @@ -1,3 +1,11 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + use color_eyre::eyre::Result; use crossterm::event::KeyEvent; use ratatui::prelude::Rect; diff --git a/node-tui/src/cli.rs b/node-tui/src/cli.rs index b0ac6f372c..ecd2fa8abc 100644 --- a/node-tui/src/cli.rs +++ b/node-tui/src/cli.rs @@ -1,3 +1,11 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + use clap::Parser; use sn_peers_acquisition::PeersArgs; diff --git a/node-tui/src/components.rs b/node-tui/src/components.rs index 4c643a663b..9e6390e872 100644 --- a/node-tui/src/components.rs +++ b/node-tui/src/components.rs @@ -1,3 +1,11 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + use color_eyre::eyre::Result; use crossterm::event::{KeyEvent, MouseEvent}; use ratatui::layout::Rect; diff --git a/node-tui/src/components/home.rs b/node-tui/src/components/home.rs index 0cf46d3030..97c1a261d7 100644 --- a/node-tui/src/components/home.rs +++ b/node-tui/src/components/home.rs @@ -1,3 +1,11 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + use color_eyre::eyre::{OptionExt, Result}; use ratatui::{prelude::*, widgets::*}; use sn_node_manager::{cmd::node::ProgressType, config::get_node_registry_path}; diff --git a/node-tui/src/config.rs b/node-tui/src/config.rs index 359bfa58e4..18766d2b6e 100644 --- a/node-tui/src/config.rs +++ b/node-tui/src/config.rs @@ -1,3 +1,11 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + use std::{collections::HashMap, path::PathBuf}; use color_eyre::eyre::Result; diff --git a/node-tui/src/main.rs b/node-tui/src/main.rs index 521e045ad6..c83609030e 100644 --- a/node-tui/src/main.rs +++ b/node-tui/src/main.rs @@ -1,4 +1,11 @@ -/// initially generated from: https://github.com/ratatui-org/templates/blob/main/component/README.md +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + pub mod action; pub mod app; pub mod cli; diff --git a/node-tui/src/mode.rs b/node-tui/src/mode.rs index 1775f94813..47a527fc24 100644 --- a/node-tui/src/mode.rs +++ b/node-tui/src/mode.rs @@ -1,3 +1,11 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + use serde::{Deserialize, Serialize}; #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] diff --git a/node-tui/src/tui.rs b/node-tui/src/tui.rs index 88da791ef4..4ddfb19a8c 100644 --- a/node-tui/src/tui.rs +++ b/node-tui/src/tui.rs @@ -1,3 +1,11 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + use std::{ ops::{Deref, DerefMut}, time::Duration, diff --git a/node-tui/src/utils.rs b/node-tui/src/utils.rs index 8798274a7a..de891be601 100644 --- a/node-tui/src/utils.rs +++ b/node-tui/src/utils.rs @@ -1,3 +1,11 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + use std::path::PathBuf; use color_eyre::eyre::Result; From 2b83f632514588520d5d98e49eb5286b3392adb0 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 1 May 2024 16:37:00 +0530 Subject: [PATCH 185/205] feat(tui): enable swtiching between multiple tabs --- node-tui/.config/config.json5 | 12 ++++- node-tui/src/action.rs | 11 ++++- node-tui/src/app.rs | 44 ++++++++++------- node-tui/src/components.rs | 1 + node-tui/src/components/home.rs | 46 +++++++++++------- node-tui/src/components/tab.rs | 85 +++++++++++++++++++++++++++++++++ node-tui/src/config.rs | 16 +++---- node-tui/src/mode.rs | 10 +++- 8 files changed, 179 insertions(+), 46 deletions(-) create mode 100644 node-tui/src/components/tab.rs diff --git a/node-tui/.config/config.json5 b/node-tui/.config/config.json5 index 7178fea801..0d4d022a5c 100644 --- a/node-tui/.config/config.json5 +++ b/node-tui/.config/config.json5 @@ -5,11 +5,21 @@ "": {"HomeActions":"StartNodes"}, // Start nodes "": {"HomeActions":"StopNode"}, // Stop nodes "up" : {"HomeActions":"PreviousTableItem"}, - "down" : {"HomeActions":"NextTableItem"}, + "down": {"HomeActions":"NextTableItem"}, + "tab": {"TabActions":"NextTab"}, + "": {"TabActions":"PreviousTab"}, "": "Quit", // Quit the application "": "Quit", // Another way to quit "": "Quit", // Yet another way to quit "": "Suspend" // Suspend the application }, + "Options": { + "tab": {"TabActions":"NextTab"}, + "": {"TabActions":"PreviousTab"}, + "": "Quit", // Quit the application + "": "Quit", // Another way to quit + "": "Quit", // Yet another way to quit + "": "Suspend" // Suspend the application + } } } diff --git a/node-tui/src/action.rs b/node-tui/src/action.rs index 99c187404f..2f4782f78c 100644 --- a/node-tui/src/action.rs +++ b/node-tui/src/action.rs @@ -6,18 +6,21 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. +use crate::mode::Scene; use serde::{Deserialize, Serialize}; use strum::Display; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Display, Deserialize)] pub enum Action { HomeActions(HomeActions), + TabActions(TabActions), + SwitchScene(Scene), + Tick, Render, Resize(u16, u16), Suspend, Resume, - StartNode, Quit, Refresh, Error(String), @@ -36,3 +39,9 @@ pub enum HomeActions { PreviousTableItem, NextTableItem, } + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Display, Deserialize)] +pub enum TabActions { + NextTab, + PreviousTab, +} diff --git a/node-tui/src/app.rs b/node-tui/src/app.rs index 3e1d3ea2ea..0b75997676 100644 --- a/node-tui/src/app.rs +++ b/node-tui/src/app.rs @@ -14,9 +14,9 @@ use tokio::sync::mpsc; use crate::{ action::Action, - components::{home::Home, Component}, + components::{home::Home, tab::Tab, Component}, config::Config, - mode::Mode, + mode::{Mode, Scene}, tui, }; @@ -28,22 +28,25 @@ pub struct App { pub should_quit: bool, pub should_suspend: bool, pub mode: Mode, + pub scene: Scene, pub last_tick_key_events: Vec, } impl App { pub fn new(tick_rate: f64, frame_rate: f64, peers_args: PeersArgs) -> Result { + let tab = Tab::default(); let home = Home::new(peers_args)?; let config = Config::new()?; - let mode = Mode::Home; + let scene = tab.get_current_scene(); Ok(Self { tick_rate, frame_rate, - components: vec![Box::new(home)], + components: vec![Box::new(home), Box::new(tab)], should_quit: false, should_suspend: false, config, - mode, + mode: Mode::Navigation, + scene, last_tick_key_events: Vec::new(), }) } @@ -75,22 +78,24 @@ impl App { tui::Event::Render => action_tx.send(Action::Render)?, tui::Event::Resize(x, y) => action_tx.send(Action::Resize(x, y))?, tui::Event::Key(key) => { - if let Some(keymap) = self.config.keybindings.get(&self.mode) { - if let Some(action) = keymap.get(&vec![key]) { - log::info!("Got action: {action:?}"); - action_tx.send(action.clone())?; - } else { - // If the key was not handled as a single key action, - // then consider it for multi-key combinations. - self.last_tick_key_events.push(key); - - // Check for multi-key combinations - if let Some(action) = keymap.get(&self.last_tick_key_events) { + if self.mode == Mode::Navigation { + if let Some(keymap) = self.config.keybindings.get(&self.scene) { + if let Some(action) = keymap.get(&vec![key]) { log::info!("Got action: {action:?}"); action_tx.send(action.clone())?; + } else { + // If the key was not handled as a single key action, + // then consider it for multi-key combinations. + self.last_tick_key_events.push(key); + + // Check for multi-key combinations + if let Some(action) = keymap.get(&self.last_tick_key_events) { + log::info!("Got action: {action:?}"); + action_tx.send(action.clone())?; + } } - } - }; + }; + } }, _ => {}, } @@ -133,6 +138,9 @@ impl App { } })?; }, + Action::SwitchScene(scene) => { + self.scene = scene; + }, _ => {}, } for component in self.components.iter_mut() { diff --git a/node-tui/src/components.rs b/node-tui/src/components.rs index 9e6390e872..eab3286c16 100644 --- a/node-tui/src/components.rs +++ b/node-tui/src/components.rs @@ -18,6 +18,7 @@ use crate::{ }; pub mod home; +pub mod tab; /// `Component` is a trait that represents a visual and interactive element of the user interface. /// Implementors of this trait can be registered with the main application loop and will be able to receive events, diff --git a/node-tui/src/components/home.rs b/node-tui/src/components/home.rs index 97c1a261d7..8673fbd385 100644 --- a/node-tui/src/components/home.rs +++ b/node-tui/src/components/home.rs @@ -6,6 +6,12 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. +use super::{Component, Frame}; +use crate::{ + action::{Action, HomeActions}, + config::Config, + mode::Scene, +}; use color_eyre::eyre::{OptionExt, Result}; use ratatui::{prelude::*, widgets::*}; use sn_node_manager::{cmd::node::ProgressType, config::get_node_registry_path}; @@ -13,24 +19,19 @@ use sn_peers_acquisition::PeersArgs; use sn_service_management::{NodeRegistry, NodeServiceData, ServiceStatus}; use tokio::sync::mpsc::{self, UnboundedSender}; -use super::{Component, Frame}; -use crate::{ - action::{Action, HomeActions}, - config::Config, -}; - #[derive(Default)] pub struct Home { action_sender: Option>, config: Config, // state + show_scene: bool, running_nodes: Vec, node_table_state: TableState, // Currently the node registry file does not support concurrent actions and thus can lead to // inconsistent state. A simple file lock or a db like file would work. lock_registry: bool, - // Network Peers + // Network Peer pub peers_args: PeersArgs, } @@ -38,6 +39,7 @@ impl Home { pub fn new(peers_args: PeersArgs) -> Result { let mut home = Self { peers_args, ..Default::default() }; home.load_node_registry()?; + home.show_scene = true; Ok(home) } } @@ -55,6 +57,10 @@ impl Component for Home { fn update(&mut self, action: Action) -> Result> { match action { + Action::SwitchScene(mode) => match mode { + Scene::Home => self.show_scene = true, + _ => self.show_scene = false, + }, Action::HomeActions(HomeActions::AddNode) => { if self.lock_registry { error!("Registry is locked, cannot add node now."); @@ -158,15 +164,22 @@ impl Component for Home { } fn draw(&mut self, f: &mut Frame<'_>, area: Rect) -> Result<()> { - // basic home layout - let home_layout = - Layout::new(Direction::Vertical, [Constraint::Min(5), Constraint::Min(3), Constraint::Max(3)]).split(area); + if !self.show_scene { + return Ok(()); + } + + // index 0 is reserved for tab + let layer_zero = Layout::new( + Direction::Vertical, + [Constraint::Max(1), Constraint::Min(5), Constraint::Min(3), Constraint::Max(3)], + ) + .split(area); // top section // f.render_widget( Paragraph::new("None").block(Block::default().title("Autonomi Node Status").borders(Borders::ALL)), - home_layout[0], + layer_zero[1], ); // Node List @@ -175,7 +188,6 @@ impl Component for Home { .iter() .filter_map(|n| { let peer_id = n.peer_id; - info!("peer_id {:?} {:?}", peer_id, n.status); if n.status == ServiceStatus::Removed { return None; } @@ -196,12 +208,14 @@ impl Component for Home { .block(Block::default().title("Running Nodes").borders(Borders::ALL)) .highlight_symbol(">"); - f.render_stateful_widget(table, home_layout[1], &mut self.node_table_state); + f.render_stateful_widget(table, layer_zero[2], &mut self.node_table_state); f.render_widget( - Paragraph::new("[A]dd node, [S]tart node, [K]ill node, [Q]uit") - .block(Block::default().title(" Key commands ").borders(Borders::ALL)), - home_layout[2], + Paragraph::new( + "[A]dd node, [S]tart node, [K]ill node, [Q]uit, [Tab] Next Page, [Shift + Tab] Previous Page", + ) + .block(Block::default().title(" Key commands ").borders(Borders::ALL)), + layer_zero[3], ); Ok(()) } diff --git a/node-tui/src/components/tab.rs b/node-tui/src/components/tab.rs new file mode 100644 index 0000000000..a4f87dfb49 --- /dev/null +++ b/node-tui/src/components/tab.rs @@ -0,0 +1,85 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use super::Component; +use crate::{ + action::{Action, TabActions}, + mode::Scene, +}; +use color_eyre::Result; +use ratatui::{ + layout::{Constraint, Direction, Layout}, + style::{Style, Stylize}, + widgets::Tabs, +}; + +pub struct Tab { + scene_list: Vec, + current_tab_index: usize, +} + +impl Default for Tab { + fn default() -> Self { + Self { scene_list: vec![Scene::Home, Scene::Options], current_tab_index: 0 } + } +} + +impl Tab { + pub fn get_current_scene(&self) -> Scene { + self.scene_list[self.current_tab_index] + } +} + +impl Component for Tab { + fn update(&mut self, action: Action) -> Result> { + let send_back = match action { + Action::TabActions(TabActions::NextTab) => { + info!(?self.current_tab_index, "Got Next tab"); + let mut new_index = self.current_tab_index + 1; + if new_index >= self.scene_list.len() { + new_index = 0; + } + self.current_tab_index = new_index; + let new_scene = self.scene_list[self.current_tab_index]; + info!(?new_scene, "Updated tab:"); + Some(Action::SwitchScene(new_scene)) + }, + + Action::TabActions(TabActions::PreviousTab) => { + info!(?self.current_tab_index, "Got PreviousTab"); + let new_index = + if self.current_tab_index == 0 { self.scene_list.len() - 1 } else { self.current_tab_index - 1 }; + self.current_tab_index = new_index; + + let new_scene = self.scene_list[self.current_tab_index]; + info!(?new_scene, "Updated tab:"); + Some(Action::SwitchScene(new_scene)) + }, + _ => None, + }; + Ok(send_back) + } + + fn draw(&mut self, f: &mut crate::tui::Frame<'_>, area: ratatui::prelude::Rect) -> Result<()> { + let layer_zero = Layout::new( + Direction::Vertical, + [Constraint::Max(1), Constraint::Min(5), Constraint::Min(3), Constraint::Max(3)], + ) + .split(area); + let tab_items = self.scene_list.iter().map(|item| format!("{item:?}")).collect::>(); + let tab = Tabs::new(tab_items) + .style(Style::default().white()) + .highlight_style(Style::default().yellow()) + .select(self.current_tab_index) + .divider("|") + .padding(" ", " "); + f.render_widget(tab, layer_zero[0]); + + Ok(()) + } +} diff --git a/node-tui/src/config.rs b/node-tui/src/config.rs index 18766d2b6e..be1e3d9caf 100644 --- a/node-tui/src/config.rs +++ b/node-tui/src/config.rs @@ -6,15 +6,13 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use std::{collections::HashMap, path::PathBuf}; - +use crate::{action::Action, mode::Scene}; use color_eyre::eyre::Result; use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; use derive_deref::{Deref, DerefMut}; use ratatui::style::{Color, Modifier, Style}; use serde::{de::Deserializer, Deserialize}; - -use crate::{action::Action, mode::Mode}; +use std::{collections::HashMap, path::PathBuf}; const CONFIG: &str = include_str!("../.config/config.json5"); @@ -83,14 +81,14 @@ impl Config { } #[derive(Clone, Debug, Default, Deref, DerefMut)] -pub struct KeyBindings(pub HashMap, Action>>); +pub struct KeyBindings(pub HashMap, Action>>); impl<'de> Deserialize<'de> for KeyBindings { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { - let parsed_map = HashMap::>::deserialize(deserializer)?; + let parsed_map = HashMap::>::deserialize(deserializer)?; let keybindings = parsed_map .into_iter() @@ -274,14 +272,14 @@ pub fn parse_key_sequence(raw: &str) -> Result, String> { } #[derive(Clone, Debug, Default, Deref, DerefMut)] -pub struct Styles(pub HashMap>); +pub struct Styles(pub HashMap>); impl<'de> Deserialize<'de> for Styles { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { - let parsed_map = HashMap::>::deserialize(deserializer)?; + let parsed_map = HashMap::>::deserialize(deserializer)?; let styles = parsed_map .into_iter() @@ -447,7 +445,7 @@ mod tests { fn test_config() -> Result<()> { let c = Config::new()?; assert_eq!( - c.keybindings.get(&Mode::Home).unwrap().get(&parse_key_sequence("").unwrap_or_default()).unwrap(), + c.keybindings.get(&Scene::Home).unwrap().get(&parse_key_sequence("").unwrap_or_default()).unwrap(), &Action::Quit ); Ok(()) diff --git a/node-tui/src/mode.rs b/node-tui/src/mode.rs index 47a527fc24..a52f0c7888 100644 --- a/node-tui/src/mode.rs +++ b/node-tui/src/mode.rs @@ -9,7 +9,15 @@ use serde::{Deserialize, Serialize}; #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum Mode { +pub enum Scene { #[default] Home, + Options, +} + +#[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Mode { + #[default] + Navigation, + Input, } From 35967299abd54d36b5f4e1f075bf49499bd2fb45 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 1 May 2024 18:12:42 +0530 Subject: [PATCH 186/205] feat(tui): implement options screen with input support --- Cargo.lock | 11 +++ node-tui/.config/config.json5 | 2 + node-tui/Cargo.toml | 1 + node-tui/src/action.rs | 3 +- node-tui/src/app.rs | 46 ++++++------ node-tui/src/components.rs | 1 + node-tui/src/components/home.rs | 2 +- node-tui/src/components/options.rs | 108 +++++++++++++++++++++++++++++ node-tui/src/components/tab.rs | 8 +-- node-tui/src/mode.rs | 4 +- 10 files changed, 158 insertions(+), 28 deletions(-) create mode 100644 node-tui/src/components/options.rs diff --git a/Cargo.lock b/Cargo.lock index 03b07eabcc..e4866d54bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4617,6 +4617,7 @@ dependencies = [ "tracing", "tracing-error", "tracing-subscriber", + "tui-input", "vergen", ] @@ -8193,6 +8194,16 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tui-input" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3e785f863a3af4c800a2a669d0b64c879b538738e352607e2624d03f868dc01" +dependencies = [ + "crossterm", + "unicode-width", +] + [[package]] name = "tungstenite" version = "0.21.0" diff --git a/node-tui/.config/config.json5 b/node-tui/.config/config.json5 index 0d4d022a5c..5b85c617ac 100644 --- a/node-tui/.config/config.json5 +++ b/node-tui/.config/config.json5 @@ -16,6 +16,8 @@ "Options": { "tab": {"TabActions":"NextTab"}, "": {"TabActions":"PreviousTab"}, + "enter": {"SwitchInputMode":"Entry"}, + // "esc": {"SwitchInputMode":"Navigation"}, // disabled because when in entry mode, keybindings are not captured "": "Quit", // Quit the application "": "Quit", // Another way to quit "": "Quit", // Yet another way to quit diff --git a/node-tui/Cargo.toml b/node-tui/Cargo.toml index a313cc989f..45687aa69e 100644 --- a/node-tui/Cargo.toml +++ b/node-tui/Cargo.toml @@ -46,6 +46,7 @@ tokio-util = "0.7.9" tracing = "0.1.37" tracing-error = "0.2.0" tracing-subscriber = { version = "0.3.17", features = ["env-filter", "serde"] } +tui-input = "0.8.0" [build-dependencies] vergen = { version = "8.2.6", features = ["build", "git", "gitoxide", "cargo"] } diff --git a/node-tui/src/action.rs b/node-tui/src/action.rs index 2f4782f78c..48fa3d4165 100644 --- a/node-tui/src/action.rs +++ b/node-tui/src/action.rs @@ -6,7 +6,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::mode::Scene; +use crate::mode::{InputMode, Scene}; use serde::{Deserialize, Serialize}; use strum::Display; @@ -15,6 +15,7 @@ pub enum Action { HomeActions(HomeActions), TabActions(TabActions), SwitchScene(Scene), + SwitchInputMode(InputMode), Tick, Render, diff --git a/node-tui/src/app.rs b/node-tui/src/app.rs index 0b75997676..0622378fc8 100644 --- a/node-tui/src/app.rs +++ b/node-tui/src/app.rs @@ -6,19 +6,18 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use color_eyre::eyre::Result; -use crossterm::event::KeyEvent; -use ratatui::prelude::Rect; -use sn_peers_acquisition::PeersArgs; -use tokio::sync::mpsc; - use crate::{ action::Action, - components::{home::Home, tab::Tab, Component}, + components::{home::Home, options::Options, tab::Tab, Component}, config::Config, - mode::{Mode, Scene}, + mode::{InputMode, Scene}, tui, }; +use color_eyre::eyre::Result; +use crossterm::event::KeyEvent; +use ratatui::prelude::Rect; +use sn_peers_acquisition::PeersArgs; +use tokio::sync::mpsc; pub struct App { pub config: Config, @@ -27,7 +26,7 @@ pub struct App { pub components: Vec>, pub should_quit: bool, pub should_suspend: bool, - pub mode: Mode, + pub input_mode: InputMode, pub scene: Scene, pub last_tick_key_events: Vec, } @@ -37,15 +36,16 @@ impl App { let tab = Tab::default(); let home = Home::new(peers_args)?; let config = Config::new()?; + let options = Options::default(); let scene = tab.get_current_scene(); Ok(Self { tick_rate, frame_rate, - components: vec![Box::new(home), Box::new(tab)], + components: vec![Box::new(home), Box::new(options), Box::new(tab)], should_quit: false, should_suspend: false, config, - mode: Mode::Navigation, + input_mode: InputMode::Navigation, scene, last_tick_key_events: Vec::new(), }) @@ -78,10 +78,10 @@ impl App { tui::Event::Render => action_tx.send(Action::Render)?, tui::Event::Resize(x, y) => action_tx.send(Action::Resize(x, y))?, tui::Event::Key(key) => { - if self.mode == Mode::Navigation { + if self.input_mode == InputMode::Navigation { if let Some(keymap) = self.config.keybindings.get(&self.scene) { if let Some(action) = keymap.get(&vec![key]) { - log::info!("Got action: {action:?}"); + info!("Got action: {action:?}"); action_tx.send(action.clone())?; } else { // If the key was not handled as a single key action, @@ -90,25 +90,26 @@ impl App { // Check for multi-key combinations if let Some(action) = keymap.get(&self.last_tick_key_events) { - log::info!("Got action: {action:?}"); + info!("Got action: {action:?}"); action_tx.send(action.clone())?; } } }; + } else if self.input_mode == InputMode::Entry { + for component in self.components.iter_mut() { + if let Some(action) = component.handle_events(Some(e.clone()))? { + action_tx.send(action)?; + } + } } }, _ => {}, } - for component in self.components.iter_mut() { - if let Some(action) = component.handle_events(Some(e.clone()))? { - action_tx.send(action)?; - } - } } while let Ok(action) = action_rx.try_recv() { if action != Action::Tick && action != Action::Render { - log::debug!("{action:?}"); + debug!("{action:?}"); } match action { Action::Tick => { @@ -139,8 +140,13 @@ impl App { })?; }, Action::SwitchScene(scene) => { + info!("Scene swtiched to: {scene:?}"); self.scene = scene; }, + Action::SwitchInputMode(mode) => { + info!("Input mode switched to: {mode:?}"); + self.input_mode = mode; + }, _ => {}, } for component in self.components.iter_mut() { diff --git a/node-tui/src/components.rs b/node-tui/src/components.rs index eab3286c16..8242b26696 100644 --- a/node-tui/src/components.rs +++ b/node-tui/src/components.rs @@ -18,6 +18,7 @@ use crate::{ }; pub mod home; +pub mod options; pub mod tab; /// `Component` is a trait that represents a visual and interactive element of the user interface. diff --git a/node-tui/src/components/home.rs b/node-tui/src/components/home.rs index 8673fbd385..41a2edf99f 100644 --- a/node-tui/src/components/home.rs +++ b/node-tui/src/components/home.rs @@ -57,7 +57,7 @@ impl Component for Home { fn update(&mut self, action: Action) -> Result> { match action { - Action::SwitchScene(mode) => match mode { + Action::SwitchScene(scene) => match scene { Scene::Home => self.show_scene = true, _ => self.show_scene = false, }, diff --git a/node-tui/src/components/options.rs b/node-tui/src/components/options.rs new file mode 100644 index 0000000000..65744d1afb --- /dev/null +++ b/node-tui/src/components/options.rs @@ -0,0 +1,108 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use super::Component; +use crate::{ + action::Action, + mode::{InputMode, Scene}, +}; +use color_eyre::Result; +use crossterm::event::{Event, KeyCode, KeyEvent}; +use ratatui::{prelude::*, widgets::*}; +use tui_input::{backend::crossterm::EventHandler, Input}; + +#[derive(Default)] +pub struct Options { + // state + show_scene: bool, + input_mode: InputMode, + input: Input, +} + +impl Component for Options { + fn handle_key_events(&mut self, key: KeyEvent) -> Result> { + // while in entry mode, keybinds are not captured, so gotta exit entry mode from here + match key.code { + KeyCode::Esc => { + return Ok(Some(Action::SwitchInputMode(InputMode::Navigation))); + }, + KeyCode::Down => { + // self.select_next_input_field(); + }, + KeyCode::Up => { + // self.select_previous_input_field(); + }, + _ => {}, + } + self.input.handle_event(&Event::Key(key)); + Ok(None) + } + + fn update(&mut self, action: Action) -> Result> { + match action { + Action::SwitchScene(scene) => match scene { + Scene::Options => self.show_scene = true, + _ => self.show_scene = false, + }, + Action::SwitchInputMode(mode) => self.input_mode = mode, + _ => {}, + }; + Ok(None) + } + + fn draw(&mut self, f: &mut crate::tui::Frame<'_>, area: Rect) -> Result<()> { + if !self.show_scene { + return Ok(()); + } + + // index 0 is reserved for tab; 2 is for keybindings + let layer_zero = + Layout::new(Direction::Vertical, [Constraint::Max(1), Constraint::Min(15), Constraint::Max(3)]).split(area); + + // break the index 1 into sub sections + let layer_one = Layout::new( + Direction::Vertical, + [ + Constraint::Length(3), + Constraint::Length(3), + Constraint::Length(3), + Constraint::Length(3), + Constraint::Length(3), + Constraint::Length(3), + ], + ) + .split(layer_zero[1]); + + let input = Paragraph::new(self.input.value()) + .style(Style::default()) + .block(Block::default().borders(Borders::ALL).title("Peer MultiAddress")); + f.render_widget(input, layer_one[0]); + let input = Paragraph::new(self.input.value()) + .style(Style::default()) + .block(Block::default().borders(Borders::ALL).title("Home Network")); + f.render_widget(input, layer_one[1]); + let input = Paragraph::new(self.input.value()) + .style(Style::default()) + .block(Block::default().borders(Borders::ALL).title("Data dir Path")); + f.render_widget(input, layer_one[2]); + let input = Paragraph::new(self.input.value()) + .style(Style::default()) + .block(Block::default().borders(Borders::ALL).title("Log dir Path")); + f.render_widget(input, layer_one[3]); + let input = Paragraph::new(self.input.value()) + .style(Style::default()) + .block(Block::default().borders(Borders::ALL).title("Node Version")); + f.render_widget(input, layer_one[4]); + let input = Paragraph::new(self.input.value()) + .style(Style::default()) + .block(Block::default().borders(Borders::ALL).title("RPC Address")); + f.render_widget(input, layer_one[5]); + + Ok(()) + } +} diff --git a/node-tui/src/components/tab.rs b/node-tui/src/components/tab.rs index a4f87dfb49..2d91bd15b2 100644 --- a/node-tui/src/components/tab.rs +++ b/node-tui/src/components/tab.rs @@ -39,25 +39,25 @@ impl Component for Tab { fn update(&mut self, action: Action) -> Result> { let send_back = match action { Action::TabActions(TabActions::NextTab) => { - info!(?self.current_tab_index, "Got Next tab"); + trace!(?self.current_tab_index, "Got Next tab"); let mut new_index = self.current_tab_index + 1; if new_index >= self.scene_list.len() { new_index = 0; } self.current_tab_index = new_index; let new_scene = self.scene_list[self.current_tab_index]; - info!(?new_scene, "Updated tab:"); + trace!(?new_scene, "Updated tab:"); Some(Action::SwitchScene(new_scene)) }, Action::TabActions(TabActions::PreviousTab) => { - info!(?self.current_tab_index, "Got PreviousTab"); + trace!(?self.current_tab_index, "Got PreviousTab"); let new_index = if self.current_tab_index == 0 { self.scene_list.len() - 1 } else { self.current_tab_index - 1 }; self.current_tab_index = new_index; let new_scene = self.scene_list[self.current_tab_index]; - info!(?new_scene, "Updated tab:"); + trace!(?new_scene, "Updated tab:"); Some(Action::SwitchScene(new_scene)) }, _ => None, diff --git a/node-tui/src/mode.rs b/node-tui/src/mode.rs index a52f0c7888..a1b1c019e6 100644 --- a/node-tui/src/mode.rs +++ b/node-tui/src/mode.rs @@ -16,8 +16,8 @@ pub enum Scene { } #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum Mode { +pub enum InputMode { #[default] Navigation, - Input, + Entry, } From e722b639dddf0c80c4231f0c262ff76b4d28f153 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 1 May 2024 19:36:19 +0530 Subject: [PATCH 187/205] chore: remove previous tab keybind --- node-tui/.config/config.json5 | 1 - node-tui/src/components/home.rs | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/node-tui/.config/config.json5 b/node-tui/.config/config.json5 index 5b85c617ac..e88ed1ed77 100644 --- a/node-tui/.config/config.json5 +++ b/node-tui/.config/config.json5 @@ -17,7 +17,6 @@ "tab": {"TabActions":"NextTab"}, "": {"TabActions":"PreviousTab"}, "enter": {"SwitchInputMode":"Entry"}, - // "esc": {"SwitchInputMode":"Navigation"}, // disabled because when in entry mode, keybindings are not captured "": "Quit", // Quit the application "": "Quit", // Another way to quit "": "Quit", // Yet another way to quit diff --git a/node-tui/src/components/home.rs b/node-tui/src/components/home.rs index 41a2edf99f..14d57e83e6 100644 --- a/node-tui/src/components/home.rs +++ b/node-tui/src/components/home.rs @@ -211,10 +211,8 @@ impl Component for Home { f.render_stateful_widget(table, layer_zero[2], &mut self.node_table_state); f.render_widget( - Paragraph::new( - "[A]dd node, [S]tart node, [K]ill node, [Q]uit, [Tab] Next Page, [Shift + Tab] Previous Page", - ) - .block(Block::default().title(" Key commands ").borders(Borders::ALL)), + Paragraph::new("[A]dd node, [S]tart node, [K]ill node, [Q]uit, [Tab] Next Page") + .block(Block::default().title(" Key commands ").borders(Borders::ALL)), layer_zero[3], ); Ok(()) From 0e19f3427f046f551cb72f8b2c27c3aacf6c7de1 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 1 May 2024 20:16:12 +0530 Subject: [PATCH 188/205] feat: popup during add or start service --- node-tui/src/components/home.rs | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/node-tui/src/components/home.rs b/node-tui/src/components/home.rs index 14d57e83e6..5df55c9d3b 100644 --- a/node-tui/src/components/home.rs +++ b/node-tui/src/components/home.rs @@ -174,6 +174,7 @@ impl Component for Home { [Constraint::Max(1), Constraint::Min(5), Constraint::Min(3), Constraint::Max(3)], ) .split(area); + let popup_area = Self::centered_rect(25, 25, area); // top section // @@ -215,6 +216,18 @@ impl Component for Home { .block(Block::default().title(" Key commands ").borders(Borders::ALL)), layer_zero[3], ); + + // popup + if self.lock_registry { + f.render_widget(Clear, popup_area); + f.render_widget( + Paragraph::new("Adding/Starting Node.. Please wait...") + .alignment(Alignment::Center) + .block(Block::default().borders(Borders::ALL).style(Style::default().bg(Color::Reset))), + popup_area, + ); + } + Ok(()) } } @@ -264,4 +277,21 @@ impl Home { fn unselect_table_item(&mut self) { self.node_table_state.select(None); } + + /// helper function to create a centered rect using up certain percentage of the available rect `r` + fn centered_rect(percent_x: u16, percent_y: u16, r: Rect) -> Rect { + let popup_layout = Layout::vertical([ + Constraint::Percentage((100 - percent_y) / 2), + Constraint::Percentage(percent_y), + Constraint::Percentage((100 - percent_y) / 2), + ]) + .split(r); + + Layout::horizontal([ + Constraint::Percentage((100 - percent_x) / 2), + Constraint::Percentage(percent_x), + Constraint::Percentage((100 - percent_x) / 2), + ]) + .split(popup_layout[1])[1] + } } From 02eef0cfb40814941e8c6b22b246d8855218c2f0 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 1 May 2024 20:32:48 +0530 Subject: [PATCH 189/205] refactor(tui): rename crate --- Cargo.lock | 2 +- Cargo.toml | 2 +- {node-tui => node-launchpad}/.config/config.json5 | 0 {node-tui => node-launchpad}/.envrc | 0 .../.github/workflows/cd.yml | 0 .../.github/workflows/ci.yml | 0 {node-tui => node-launchpad}/.gitignore | 0 {node-tui => node-launchpad}/.rustfmt.toml | 0 {node-tui => node-launchpad}/Cargo.toml | 12 +++++++----- {node-tui => node-launchpad}/LICENSE | 0 {node-tui => node-launchpad}/README.md | 2 +- {node-tui => node-launchpad}/build.rs | 0 {node-tui => node-launchpad}/src/action.rs | 0 {node-tui => node-launchpad}/src/app.rs | 0 {node-tui => node-launchpad}/src/cli.rs | 0 {node-tui => node-launchpad}/src/components.rs | 0 {node-tui => node-launchpad}/src/components/home.rs | 6 ++---- .../src/components/options.rs | 0 {node-tui => node-launchpad}/src/components/tab.rs | 0 {node-tui => node-launchpad}/src/config.rs | 0 {node-tui => node-launchpad}/src/main.rs | 0 {node-tui => node-launchpad}/src/mode.rs | 0 {node-tui => node-launchpad}/src/tui.rs | 0 {node-tui => node-launchpad}/src/utils.rs | 0 24 files changed, 12 insertions(+), 12 deletions(-) rename {node-tui => node-launchpad}/.config/config.json5 (100%) rename {node-tui => node-launchpad}/.envrc (100%) rename {node-tui => node-launchpad}/.github/workflows/cd.yml (100%) rename {node-tui => node-launchpad}/.github/workflows/ci.yml (100%) rename {node-tui => node-launchpad}/.gitignore (100%) rename {node-tui => node-launchpad}/.rustfmt.toml (100%) rename {node-tui => node-launchpad}/Cargo.toml (85%) rename {node-tui => node-launchpad}/LICENSE (100%) rename {node-tui => node-launchpad}/README.md (74%) rename {node-tui => node-launchpad}/build.rs (100%) rename {node-tui => node-launchpad}/src/action.rs (100%) rename {node-tui => node-launchpad}/src/app.rs (100%) rename {node-tui => node-launchpad}/src/cli.rs (100%) rename {node-tui => node-launchpad}/src/components.rs (100%) rename {node-tui => node-launchpad}/src/components/home.rs (97%) rename {node-tui => node-launchpad}/src/components/options.rs (100%) rename {node-tui => node-launchpad}/src/components/tab.rs (100%) rename {node-tui => node-launchpad}/src/config.rs (100%) rename {node-tui => node-launchpad}/src/main.rs (100%) rename {node-tui => node-launchpad}/src/mode.rs (100%) rename {node-tui => node-launchpad}/src/tui.rs (100%) rename {node-tui => node-launchpad}/src/utils.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index e4866d54bb..f98f31f7d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4586,7 +4586,7 @@ dependencies = [ ] [[package]] -name = "node-tui" +name = "node-launchpad" version = "0.1.0" dependencies = [ "better-panic", diff --git a/Cargo.toml b/Cargo.toml index d3eeb6d517..9998e149c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] resolver = "2" members = [ + "node-launchpad", "sn_auditor", "sn_build_info", "sn_cli", @@ -19,7 +20,6 @@ members = [ "sn_transfers", "test_utils", "token_supplies", - "node-tui" ] [workspace.lints.rust] diff --git a/node-tui/.config/config.json5 b/node-launchpad/.config/config.json5 similarity index 100% rename from node-tui/.config/config.json5 rename to node-launchpad/.config/config.json5 diff --git a/node-tui/.envrc b/node-launchpad/.envrc similarity index 100% rename from node-tui/.envrc rename to node-launchpad/.envrc diff --git a/node-tui/.github/workflows/cd.yml b/node-launchpad/.github/workflows/cd.yml similarity index 100% rename from node-tui/.github/workflows/cd.yml rename to node-launchpad/.github/workflows/cd.yml diff --git a/node-tui/.github/workflows/ci.yml b/node-launchpad/.github/workflows/ci.yml similarity index 100% rename from node-tui/.github/workflows/ci.yml rename to node-launchpad/.github/workflows/ci.yml diff --git a/node-tui/.gitignore b/node-launchpad/.gitignore similarity index 100% rename from node-tui/.gitignore rename to node-launchpad/.gitignore diff --git a/node-tui/.rustfmt.toml b/node-launchpad/.rustfmt.toml similarity index 100% rename from node-tui/.rustfmt.toml rename to node-launchpad/.rustfmt.toml diff --git a/node-tui/Cargo.toml b/node-launchpad/Cargo.toml similarity index 85% rename from node-tui/Cargo.toml rename to node-launchpad/Cargo.toml index 45687aa69e..139ccb9621 100644 --- a/node-tui/Cargo.toml +++ b/node-launchpad/Cargo.toml @@ -1,13 +1,15 @@ [package] -name = "node-tui" +authors = ["MaidSafe Developers "] +description = "Node Launchpad" +name = "node-launchpad" version = "0.1.0" edition = "2021" -description = "Terminal interface for autonomi node management" - -authors = ["Josh Wilson "] +license = "GPL-3.0" +homepage = "https://maidsafe.net" +readme = "README.md" +repository = "https://github.com/maidsafe/safe_network" build = "build.rs" - [dependencies] sn_peers_acquisition = { verison = "0.2.10", path = "../sn_peers_acquisition", features = [ "network-contacts", diff --git a/node-tui/LICENSE b/node-launchpad/LICENSE similarity index 100% rename from node-tui/LICENSE rename to node-launchpad/LICENSE diff --git a/node-tui/README.md b/node-launchpad/README.md similarity index 74% rename from node-tui/README.md rename to node-launchpad/README.md index 73c34ec0a5..6eb41bcfca 100644 --- a/node-tui/README.md +++ b/node-launchpad/README.md @@ -1,3 +1,3 @@ -# node-tui +# Node Launchpad Terminal interface for autonomi node management diff --git a/node-tui/build.rs b/node-launchpad/build.rs similarity index 100% rename from node-tui/build.rs rename to node-launchpad/build.rs diff --git a/node-tui/src/action.rs b/node-launchpad/src/action.rs similarity index 100% rename from node-tui/src/action.rs rename to node-launchpad/src/action.rs diff --git a/node-tui/src/app.rs b/node-launchpad/src/app.rs similarity index 100% rename from node-tui/src/app.rs rename to node-launchpad/src/app.rs diff --git a/node-tui/src/cli.rs b/node-launchpad/src/cli.rs similarity index 100% rename from node-tui/src/cli.rs rename to node-launchpad/src/cli.rs diff --git a/node-tui/src/components.rs b/node-launchpad/src/components.rs similarity index 100% rename from node-tui/src/components.rs rename to node-launchpad/src/components.rs diff --git a/node-tui/src/components/home.rs b/node-launchpad/src/components/home.rs similarity index 97% rename from node-tui/src/components/home.rs rename to node-launchpad/src/components/home.rs index 5df55c9d3b..075f0df470 100644 --- a/node-tui/src/components/home.rs +++ b/node-launchpad/src/components/home.rs @@ -14,10 +14,10 @@ use crate::{ }; use color_eyre::eyre::{OptionExt, Result}; use ratatui::{prelude::*, widgets::*}; -use sn_node_manager::{cmd::node::ProgressType, config::get_node_registry_path}; +use sn_node_manager::config::get_node_registry_path; use sn_peers_acquisition::PeersArgs; use sn_service_management::{NodeRegistry, NodeServiceData, ServiceStatus}; -use tokio::sync::mpsc::{self, UnboundedSender}; +use tokio::sync::mpsc::{UnboundedSender}; #[derive(Default)] pub struct Home { @@ -69,7 +69,6 @@ impl Component for Home { info!("Adding a new node service"); let peers = self.peers_args.clone(); - let (progress_sender, _) = mpsc::channel::(1); let action_sender = self.get_actions_sender()?; self.lock_registry = true; @@ -90,7 +89,6 @@ impl Component for Home { None, None, sn_node_manager::VerbosityLevel::Minimal, - progress_sender, ) .await { diff --git a/node-tui/src/components/options.rs b/node-launchpad/src/components/options.rs similarity index 100% rename from node-tui/src/components/options.rs rename to node-launchpad/src/components/options.rs diff --git a/node-tui/src/components/tab.rs b/node-launchpad/src/components/tab.rs similarity index 100% rename from node-tui/src/components/tab.rs rename to node-launchpad/src/components/tab.rs diff --git a/node-tui/src/config.rs b/node-launchpad/src/config.rs similarity index 100% rename from node-tui/src/config.rs rename to node-launchpad/src/config.rs diff --git a/node-tui/src/main.rs b/node-launchpad/src/main.rs similarity index 100% rename from node-tui/src/main.rs rename to node-launchpad/src/main.rs diff --git a/node-tui/src/mode.rs b/node-launchpad/src/mode.rs similarity index 100% rename from node-tui/src/mode.rs rename to node-launchpad/src/mode.rs diff --git a/node-tui/src/tui.rs b/node-launchpad/src/tui.rs similarity index 100% rename from node-tui/src/tui.rs rename to node-launchpad/src/tui.rs diff --git a/node-tui/src/utils.rs b/node-launchpad/src/utils.rs similarity index 100% rename from node-tui/src/utils.rs rename to node-launchpad/src/utils.rs From a4efbee09400ddabeee859297f79d4fb72bc8379 Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Wed, 1 May 2024 21:13:56 +0530 Subject: [PATCH 190/205] refactor: rebased and removed custom rustfmt --- node-launchpad/.github/workflows/cd.yml | 152 ----------------------- node-launchpad/.github/workflows/ci.yml | 63 ---------- node-launchpad/.rustfmt.toml | 16 --- node-launchpad/build.rs | 5 +- node-launchpad/src/app.rs | 32 +++-- node-launchpad/src/components/home.rs | 119 +++++++++++++----- node-launchpad/src/components/options.rs | 29 +++-- node-launchpad/src/components/tab.rs | 29 +++-- node-launchpad/src/config.rs | 119 +++++++++++++----- node-launchpad/src/tui.rs | 16 ++- node-launchpad/src/utils.rs | 37 ++++-- 11 files changed, 283 insertions(+), 334 deletions(-) delete mode 100644 node-launchpad/.github/workflows/cd.yml delete mode 100644 node-launchpad/.github/workflows/ci.yml delete mode 100644 node-launchpad/.rustfmt.toml diff --git a/node-launchpad/.github/workflows/cd.yml b/node-launchpad/.github/workflows/cd.yml deleted file mode 100644 index f7c06cd77e..0000000000 --- a/node-launchpad/.github/workflows/cd.yml +++ /dev/null @@ -1,152 +0,0 @@ -name: CD # Continuous Deployment - -on: - push: - tags: - - '[v]?[0-9]+.[0-9]+.[0-9]+' - -jobs: - publish: - - name: Publishing for ${{ matrix.os }} - runs-on: ${{ matrix.os }} - - strategy: - matrix: - include: - - os: macos-latest - os-name: macos - target: x86_64-apple-darwin - architecture: x86_64 - binary-postfix: "" - binary-name: node-tui - use-cross: false - - os: macos-latest - os-name: macos - target: aarch64-apple-darwin - architecture: arm64 - binary-postfix: "" - use-cross: false - binary-name: node-tui - - os: ubuntu-latest - os-name: linux - target: x86_64-unknown-linux-gnu - architecture: x86_64 - binary-postfix: "" - use-cross: false - binary-name: node-tui - - os: windows-latest - os-name: windows - target: x86_64-pc-windows-msvc - architecture: x86_64 - binary-postfix: ".exe" - use-cross: false - binary-name: node-tui - - os: ubuntu-latest - os-name: linux - target: aarch64-unknown-linux-gnu - architecture: arm64 - binary-postfix: "" - use-cross: true - binary-name: node-tui - - os: ubuntu-latest - os-name: linux - target: i686-unknown-linux-gnu - architecture: i686 - binary-postfix: "" - use-cross: true - binary-name: node-tui - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Install Rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - target: ${{ matrix.target }} - - profile: minimal - override: true - - uses: Swatinem/rust-cache@v2 - - name: Cargo build - uses: actions-rs/cargo@v1 - with: - command: build - - use-cross: ${{ matrix.use-cross }} - - toolchain: stable - - args: --release --target ${{ matrix.target }} - - - - name: install strip command - shell: bash - run: | - - if [[ ${{ matrix.target }} == aarch64-unknown-linux-gnu ]]; then - - sudo apt update - sudo apt-get install -y binutils-aarch64-linux-gnu - fi - - name: Packaging final binary - shell: bash - run: | - - cd target/${{ matrix.target }}/release - - - ####### reduce binary size by removing debug symbols ####### - - BINARY_NAME=${{ matrix.binary-name }}${{ matrix.binary-postfix }} - if [[ ${{ matrix.target }} == aarch64-unknown-linux-gnu ]]; then - - GCC_PREFIX="aarch64-linux-gnu-" - else - GCC_PREFIX="" - fi - "$GCC_PREFIX"strip $BINARY_NAME - - ########## create tar.gz ########## - - RELEASE_NAME=${{ matrix.binary-name }}-${GITHUB_REF/refs\/tags\//}-${{ matrix.os-name }}-${{ matrix.architecture }} - - tar czvf $RELEASE_NAME.tar.gz $BINARY_NAME - - ########## create sha256 ########## - - if [[ ${{ runner.os }} == 'Windows' ]]; then - - certutil -hashfile $RELEASE_NAME.tar.gz sha256 | grep -E [A-Fa-f0-9]{64} > $RELEASE_NAME.sha256 - else - shasum -a 256 $RELEASE_NAME.tar.gz > $RELEASE_NAME.sha256 - fi - - name: Releasing assets - uses: softprops/action-gh-release@v1 - with: - files: | - - target/${{ matrix.target }}/release/${{ matrix.binary-name }}-*.tar.gz - target/${{ matrix.target }}/release/${{ matrix.binary-name }}-*.sha256 - - env: - - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - publish-cargo: - name: Publishing to Cargo - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - run: cargo publish - env: - - CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} - diff --git a/node-launchpad/.github/workflows/ci.yml b/node-launchpad/.github/workflows/ci.yml deleted file mode 100644 index c64fbee9e3..0000000000 --- a/node-launchpad/.github/workflows/ci.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: CI # Continuous Integration - -on: - push: - branches: - - main - pull_request: - -jobs: - - test: - name: Test Suite - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@nightly - - uses: Swatinem/rust-cache@v2 - - name: Run tests - run: cargo test --all-features --workspace - - rustfmt: - name: Rustfmt - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@nightly - with: - components: rustfmt - - uses: Swatinem/rust-cache@v2 - - name: Check formatting - run: cargo fmt --all --check - - clippy: - name: Clippy - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@nightly - with: - components: clippy - - uses: Swatinem/rust-cache@v2 - - name: Clippy check - run: cargo clippy --all-targets --all-features --workspace -- -D warnings - - docs: - name: Docs - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@nightly - - uses: Swatinem/rust-cache@v2 - - name: Check documentation - env: - RUSTDOCFLAGS: -D warnings - run: cargo doc --no-deps --document-private-items --all-features --workspace --examples diff --git a/node-launchpad/.rustfmt.toml b/node-launchpad/.rustfmt.toml deleted file mode 100644 index 491199d2b9..0000000000 --- a/node-launchpad/.rustfmt.toml +++ /dev/null @@ -1,16 +0,0 @@ -max_width = 120 -use_small_heuristics = "Max" -empty_item_single_line = false -force_multiline_blocks = true -format_code_in_doc_comments = true -match_block_trailing_comma = true -imports_granularity = "Crate" -normalize_comments = true -normalize_doc_attributes = true -overflow_delimited_expr = true -reorder_impl_items = true -reorder_imports = true -group_imports = "StdExternalCrate" -tab_spaces = 4 -use_field_init_shorthand = true -use_try_shorthand = true diff --git a/node-launchpad/build.rs b/node-launchpad/build.rs index 23757b5cbd..17db54bab1 100644 --- a/node-launchpad/build.rs +++ b/node-launchpad/build.rs @@ -7,6 +7,9 @@ // permissions and limitations relating to use of the SAFE Network Software. fn main() -> Result<(), Box> { - vergen::EmitBuilder::builder().all_build().all_git().emit()?; + vergen::EmitBuilder::builder() + .all_build() + .all_git() + .emit()?; Ok(()) } diff --git a/node-launchpad/src/app.rs b/node-launchpad/src/app.rs index 0622378fc8..8ab1343d39 100644 --- a/node-launchpad/src/app.rs +++ b/node-launchpad/src/app.rs @@ -54,7 +54,9 @@ impl App { pub async fn run(&mut self) -> Result<()> { let (action_tx, mut action_rx) = mpsc::unbounded_channel(); - let mut tui = tui::Tui::new()?.tick_rate(self.tick_rate).frame_rate(self.frame_rate); + let mut tui = tui::Tui::new()? + .tick_rate(self.tick_rate) + .frame_rate(self.frame_rate); // tui.mouse(true); tui.enter()?; @@ -102,8 +104,8 @@ impl App { } } } - }, - _ => {}, + } + _ => {} } } @@ -114,7 +116,7 @@ impl App { match action { Action::Tick => { self.last_tick_key_events.drain(..); - }, + } Action::Quit => self.should_quit = true, Action::Suspend => self.should_suspend = true, Action::Resume => self.should_suspend = false, @@ -124,30 +126,34 @@ impl App { for component in self.components.iter_mut() { let r = component.draw(f, f.size()); if let Err(e) = r { - action_tx.send(Action::Error(format!("Failed to draw: {:?}", e))).unwrap(); + action_tx + .send(Action::Error(format!("Failed to draw: {:?}", e))) + .unwrap(); } } })?; - }, + } Action::Render => { tui.draw(|f| { for component in self.components.iter_mut() { let r = component.draw(f, f.size()); if let Err(e) = r { - action_tx.send(Action::Error(format!("Failed to draw: {:?}", e))).unwrap(); + action_tx + .send(Action::Error(format!("Failed to draw: {:?}", e))) + .unwrap(); } } })?; - }, + } Action::SwitchScene(scene) => { info!("Scene swtiched to: {scene:?}"); self.scene = scene; - }, + } Action::SwitchInputMode(mode) => { info!("Input mode switched to: {mode:?}"); self.input_mode = mode; - }, - _ => {}, + } + _ => {} } for component in self.components.iter_mut() { if let Some(action) = component.update(action.clone())? { @@ -158,7 +164,9 @@ impl App { if self.should_suspend { tui.suspend()?; action_tx.send(Action::Resume)?; - tui = tui::Tui::new()?.tick_rate(self.tick_rate).frame_rate(self.frame_rate); + tui = tui::Tui::new()? + .tick_rate(self.tick_rate) + .frame_rate(self.frame_rate); // tui.mouse(true); tui.enter()?; } else if self.should_quit { diff --git a/node-launchpad/src/components/home.rs b/node-launchpad/src/components/home.rs index 075f0df470..d59edd27e9 100644 --- a/node-launchpad/src/components/home.rs +++ b/node-launchpad/src/components/home.rs @@ -17,7 +17,7 @@ use ratatui::{prelude::*, widgets::*}; use sn_node_manager::config::get_node_registry_path; use sn_peers_acquisition::PeersArgs; use sn_service_management::{NodeRegistry, NodeServiceData, ServiceStatus}; -use tokio::sync::mpsc::{UnboundedSender}; +use tokio::sync::mpsc::UnboundedSender; #[derive(Default)] pub struct Home { @@ -37,7 +37,10 @@ pub struct Home { impl Home { pub fn new(peers_args: PeersArgs) -> Result { - let mut home = Self { peers_args, ..Default::default() }; + let mut home = Self { + peers_args, + ..Default::default() + }; home.load_node_registry()?; home.show_scene = true; Ok(home) @@ -77,7 +80,8 @@ impl Component for Home { None, None, None, - true, + false, + false, None, None, None, @@ -96,11 +100,13 @@ impl Component for Home { } info!("Successfully added service"); // todo: need to handle these properly? - if let Err(err) = action_sender.send(Action::HomeActions(HomeActions::AddNodeCompleted)) { + if let Err(err) = + action_sender.send(Action::HomeActions(HomeActions::AddNodeCompleted)) + { error!("Error while sending action: {err:?}"); } }); - }, + } Action::HomeActions(HomeActions::StartNodes) => { if self.lock_registry { error!("Registry is locked. Cannot start node now."); @@ -111,18 +117,24 @@ impl Component for Home { self.lock_registry = true; tokio::task::spawn_local(async move { - if let Err(err) = - sn_node_manager::cmd::node::start(1, vec![], vec![], sn_node_manager::VerbosityLevel::Minimal) - .await + if let Err(err) = sn_node_manager::cmd::node::start( + 1, + vec![], + vec![], + sn_node_manager::VerbosityLevel::Minimal, + ) + .await { error!("Error while starting services {err:?}"); } - if let Err(err) = action_sender.send(Action::HomeActions(HomeActions::StartNodesCompleted)) { + if let Err(err) = + action_sender.send(Action::HomeActions(HomeActions::StartNodesCompleted)) + { error!("Error while sending action: {err:?}"); } info!("Successfully started services"); }); - }, + } Action::HomeActions(HomeActions::StopNode) => { if self.lock_registry { error!("Registry is locked. Cannot stop node now."); @@ -133,30 +145,36 @@ impl Component for Home { self.lock_registry = true; tokio::task::spawn_local(async move { - if let Err(err) = - sn_node_manager::cmd::node::stop(vec![], vec![], sn_node_manager::VerbosityLevel::Minimal).await + if let Err(err) = sn_node_manager::cmd::node::stop( + vec![], + vec![], + sn_node_manager::VerbosityLevel::Minimal, + ) + .await { error!("Error while stopping services {err:?}"); } - if let Err(err) = action_sender.send(Action::HomeActions(HomeActions::StopNodeCompleted)) { + if let Err(err) = + action_sender.send(Action::HomeActions(HomeActions::StopNodeCompleted)) + { error!("Error while sending action: {err:?}"); } info!("Successfully stopped services"); }); - }, + } Action::HomeActions(HomeActions::AddNodeCompleted) | Action::HomeActions(HomeActions::StartNodesCompleted) | Action::HomeActions(HomeActions::StopNodeCompleted) => { self.lock_registry = false; self.load_node_registry()?; - }, + } Action::HomeActions(HomeActions::PreviousTableItem) => { self.select_previous_table_item(); - }, + } Action::HomeActions(HomeActions::NextTableItem) => { self.select_next_table_item(); - }, - _ => {}, + } + _ => {} } Ok(None) } @@ -169,7 +187,12 @@ impl Component for Home { // index 0 is reserved for tab let layer_zero = Layout::new( Direction::Vertical, - [Constraint::Max(1), Constraint::Min(5), Constraint::Min(3), Constraint::Max(3)], + [ + Constraint::Max(1), + Constraint::Min(5), + Constraint::Min(3), + Constraint::Max(3), + ], ) .split(area); let popup_area = Self::centered_rect(25, 25, area); @@ -177,7 +200,11 @@ impl Component for Home { // top section // f.render_widget( - Paragraph::new("None").block(Block::default().title("Autonomi Node Status").borders(Borders::ALL)), + Paragraph::new("None").block( + Block::default() + .title("Autonomi Node Status") + .borders(Borders::ALL), + ), layer_zero[1], ); @@ -199,19 +226,34 @@ impl Component for Home { }) .collect(); - let widths = [Constraint::Max(15), Constraint::Min(30), Constraint::Max(10)]; + let widths = [ + Constraint::Max(15), + Constraint::Min(30), + Constraint::Max(10), + ]; let table = Table::new(rows, widths) .column_spacing(2) - .header(Row::new(vec!["Service", "PeerId", "Status"]).style(Style::new().bold()).bottom_margin(1)) + .header( + Row::new(vec!["Service", "PeerId", "Status"]) + .style(Style::new().bold()) + .bottom_margin(1), + ) .highlight_style(Style::new().reversed()) - .block(Block::default().title("Running Nodes").borders(Borders::ALL)) + .block( + Block::default() + .title("Running Nodes") + .borders(Borders::ALL), + ) .highlight_symbol(">"); f.render_stateful_widget(table, layer_zero[2], &mut self.node_table_state); f.render_widget( - Paragraph::new("[A]dd node, [S]tart node, [K]ill node, [Q]uit, [Tab] Next Page") - .block(Block::default().title(" Key commands ").borders(Borders::ALL)), + Paragraph::new("[A]dd node, [S]tart node, [K]ill node, [Q]uit, [Tab] Next Page").block( + Block::default() + .title(" Key commands ") + .borders(Borders::ALL), + ), layer_zero[3], ); @@ -221,7 +263,11 @@ impl Component for Home { f.render_widget( Paragraph::new("Adding/Starting Node.. Please wait...") .alignment(Alignment::Center) - .block(Block::default().borders(Borders::ALL).style(Style::default().bg(Color::Reset))), + .block( + Block::default() + .borders(Borders::ALL) + .style(Style::default().bg(Color::Reset)), + ), popup_area, ); } @@ -232,14 +278,22 @@ impl Component for Home { impl Home { fn get_actions_sender(&self) -> Result> { - self.action_sender.clone().ok_or_eyre("Action sender not registered") + self.action_sender + .clone() + .ok_or_eyre("Action sender not registered") } fn load_node_registry(&mut self) -> Result<()> { let node_registry = NodeRegistry::load(&get_node_registry_path()?)?; - self.running_nodes = - node_registry.nodes.into_iter().filter(|node| node.status != ServiceStatus::Removed).collect(); - info!("Loaded node registry. Runnign nodes: {:?}", self.running_nodes.len()); + self.running_nodes = node_registry + .nodes + .into_iter() + .filter(|node| node.status != ServiceStatus::Removed) + .collect(); + info!( + "Loaded node registry. Runnign nodes: {:?}", + self.running_nodes.len() + ); Ok(()) } @@ -252,7 +306,7 @@ impl Home { } else { i + 1 } - }, + } None => 0, }; self.node_table_state.select(Some(i)); @@ -266,12 +320,13 @@ impl Home { } else { i - 1 } - }, + } None => 0, }; self.node_table_state.select(Some(i)); } + #[allow(dead_code)] fn unselect_table_item(&mut self) { self.node_table_state.select(None); } diff --git a/node-launchpad/src/components/options.rs b/node-launchpad/src/components/options.rs index 65744d1afb..6c3123c4ec 100644 --- a/node-launchpad/src/components/options.rs +++ b/node-launchpad/src/components/options.rs @@ -30,14 +30,14 @@ impl Component for Options { match key.code { KeyCode::Esc => { return Ok(Some(Action::SwitchInputMode(InputMode::Navigation))); - }, + } KeyCode::Down => { // self.select_next_input_field(); - }, + } KeyCode::Up => { // self.select_previous_input_field(); - }, - _ => {}, + } + _ => {} } self.input.handle_event(&Event::Key(key)); Ok(None) @@ -50,7 +50,7 @@ impl Component for Options { _ => self.show_scene = false, }, Action::SwitchInputMode(mode) => self.input_mode = mode, - _ => {}, + _ => {} }; Ok(None) } @@ -61,8 +61,11 @@ impl Component for Options { } // index 0 is reserved for tab; 2 is for keybindings - let layer_zero = - Layout::new(Direction::Vertical, [Constraint::Max(1), Constraint::Min(15), Constraint::Max(3)]).split(area); + let layer_zero = Layout::new( + Direction::Vertical, + [Constraint::Max(1), Constraint::Min(15), Constraint::Max(3)], + ) + .split(area); // break the index 1 into sub sections let layer_one = Layout::new( @@ -80,7 +83,11 @@ impl Component for Options { let input = Paragraph::new(self.input.value()) .style(Style::default()) - .block(Block::default().borders(Borders::ALL).title("Peer MultiAddress")); + .block( + Block::default() + .borders(Borders::ALL) + .title("Peer MultiAddress"), + ); f.render_widget(input, layer_one[0]); let input = Paragraph::new(self.input.value()) .style(Style::default()) @@ -88,7 +95,11 @@ impl Component for Options { f.render_widget(input, layer_one[1]); let input = Paragraph::new(self.input.value()) .style(Style::default()) - .block(Block::default().borders(Borders::ALL).title("Data dir Path")); + .block( + Block::default() + .borders(Borders::ALL) + .title("Data dir Path"), + ); f.render_widget(input, layer_one[2]); let input = Paragraph::new(self.input.value()) .style(Style::default()) diff --git a/node-launchpad/src/components/tab.rs b/node-launchpad/src/components/tab.rs index 2d91bd15b2..a3f03c7684 100644 --- a/node-launchpad/src/components/tab.rs +++ b/node-launchpad/src/components/tab.rs @@ -25,7 +25,10 @@ pub struct Tab { impl Default for Tab { fn default() -> Self { - Self { scene_list: vec![Scene::Home, Scene::Options], current_tab_index: 0 } + Self { + scene_list: vec![Scene::Home, Scene::Options], + current_tab_index: 0, + } } } @@ -48,18 +51,21 @@ impl Component for Tab { let new_scene = self.scene_list[self.current_tab_index]; trace!(?new_scene, "Updated tab:"); Some(Action::SwitchScene(new_scene)) - }, + } Action::TabActions(TabActions::PreviousTab) => { trace!(?self.current_tab_index, "Got PreviousTab"); - let new_index = - if self.current_tab_index == 0 { self.scene_list.len() - 1 } else { self.current_tab_index - 1 }; + let new_index = if self.current_tab_index == 0 { + self.scene_list.len() - 1 + } else { + self.current_tab_index - 1 + }; self.current_tab_index = new_index; let new_scene = self.scene_list[self.current_tab_index]; trace!(?new_scene, "Updated tab:"); Some(Action::SwitchScene(new_scene)) - }, + } _ => None, }; Ok(send_back) @@ -68,10 +74,19 @@ impl Component for Tab { fn draw(&mut self, f: &mut crate::tui::Frame<'_>, area: ratatui::prelude::Rect) -> Result<()> { let layer_zero = Layout::new( Direction::Vertical, - [Constraint::Max(1), Constraint::Min(5), Constraint::Min(3), Constraint::Max(3)], + [ + Constraint::Max(1), + Constraint::Min(5), + Constraint::Min(3), + Constraint::Max(3), + ], ) .split(area); - let tab_items = self.scene_list.iter().map(|item| format!("{item:?}")).collect::>(); + let tab_items = self + .scene_list + .iter() + .map(|item| format!("{item:?}")) + .collect::>(); let tab = Tabs::new(tab_items) .style(Style::default().white()) .highlight_style(Style::default().yellow()) diff --git a/node-launchpad/src/config.rs b/node-launchpad/src/config.rs index be1e3d9caf..ee8cde7783 100644 --- a/node-launchpad/src/config.rs +++ b/node-launchpad/src/config.rs @@ -52,7 +52,11 @@ impl Config { ]; let mut found_config = false; for (file, format) in &config_files { - builder = builder.add_source(config::File::from(config_dir.join(file)).format(*format).required(false)); + builder = builder.add_source( + config::File::from(config_dir.join(file)) + .format(*format) + .required(false), + ); if config_dir.join(file).exists() { found_config = true } @@ -66,13 +70,17 @@ impl Config { for (mode, default_bindings) in default_config.keybindings.iter() { let user_bindings = cfg.keybindings.entry(*mode).or_default(); for (key, cmd) in default_bindings.iter() { - user_bindings.entry(key.clone()).or_insert_with(|| cmd.clone()); + user_bindings + .entry(key.clone()) + .or_insert_with(|| cmd.clone()); } } for (mode, default_styles) in default_config.styles.iter() { let user_styles = cfg.styles.entry(*mode).or_default(); for (style_key, style) in default_styles.iter() { - user_styles.entry(style_key.clone()).or_insert_with(|| *style); + user_styles + .entry(style_key.clone()) + .or_insert_with(|| *style); } } @@ -93,8 +101,10 @@ impl<'de> Deserialize<'de> for KeyBindings { let keybindings = parsed_map .into_iter() .map(|(mode, inner_map)| { - let converted_inner_map = - inner_map.into_iter().map(|(key_str, cmd)| (parse_key_sequence(&key_str).unwrap(), cmd)).collect(); + let converted_inner_map = inner_map + .into_iter() + .map(|(key_str, cmd)| (parse_key_sequence(&key_str).unwrap(), cmd)) + .collect(); (mode, converted_inner_map) }) .collect(); @@ -118,15 +128,15 @@ fn extract_modifiers(raw: &str) -> (&str, KeyModifiers) { rest if rest.starts_with("ctrl-") => { modifiers.insert(KeyModifiers::CONTROL); current = &rest[5..]; - }, + } rest if rest.starts_with("alt-") => { modifiers.insert(KeyModifiers::ALT); current = &rest[4..]; - }, + } rest if rest.starts_with("shift-") => { modifiers.insert(KeyModifiers::SHIFT); current = &rest[6..]; - }, + } _ => break, // break out of the loop if no known prefix is detected }; } @@ -134,7 +144,10 @@ fn extract_modifiers(raw: &str) -> (&str, KeyModifiers) { (current, modifiers) } -fn parse_key_code_with_modifiers(raw: &str, mut modifiers: KeyModifiers) -> Result { +fn parse_key_code_with_modifiers( + raw: &str, + mut modifiers: KeyModifiers, +) -> Result { let c = match raw { "esc" => KeyCode::Esc, "enter" => KeyCode::Enter, @@ -149,7 +162,7 @@ fn parse_key_code_with_modifiers(raw: &str, mut modifiers: KeyModifiers) -> Resu "backtab" => { modifiers.insert(KeyModifiers::SHIFT); KeyCode::BackTab - }, + } "backspace" => KeyCode::Backspace, "delete" => KeyCode::Delete, "insert" => KeyCode::Insert, @@ -175,7 +188,7 @@ fn parse_key_code_with_modifiers(raw: &str, mut modifiers: KeyModifiers) -> Resu c = c.to_ascii_uppercase(); } KeyCode::Char(c) - }, + } _ => return Err(format!("Unable to parse {raw}")), }; Ok(KeyEvent::new(c, modifiers)) @@ -201,12 +214,12 @@ pub fn key_event_to_string(key_event: &KeyEvent) -> String { KeyCode::F(c) => { char = format!("f({c})"); &char - }, + } KeyCode::Char(' ') => "space", KeyCode::Char(c) => { char = c.to_string(); &char - }, + } KeyCode::Esc => "esc", KeyCode::Null => "", KeyCode::CapsLock => "", @@ -284,8 +297,10 @@ impl<'de> Deserialize<'de> for Styles { let styles = parsed_map .into_iter() .map(|(mode, inner_map)| { - let converted_inner_map = - inner_map.into_iter().map(|(str, style)| (str, parse_style(&style))).collect(); + let converted_inner_map = inner_map + .into_iter() + .map(|(str, style)| (str, parse_style(&style))) + .collect(); (mode, converted_inner_map) }) .collect(); @@ -295,7 +310,8 @@ impl<'de> Deserialize<'de> for Styles { } pub fn parse_style(line: &str) -> Style { - let (foreground, background) = line.split_at(line.to_lowercase().find("on ").unwrap_or(line.len())); + let (foreground, background) = + line.split_at(line.to_lowercase().find("on ").unwrap_or(line.len())); let foreground = process_color_string(foreground); let background = process_color_string(&background.replace("on ", "")); @@ -337,13 +353,22 @@ fn parse_color(s: &str) -> Option { let s = s.trim_end(); if s.contains("bright color") { let s = s.trim_start_matches("bright "); - let c = s.trim_start_matches("color").parse::().unwrap_or_default(); + let c = s + .trim_start_matches("color") + .parse::() + .unwrap_or_default(); Some(Color::Indexed(c.wrapping_shl(8))) } else if s.contains("color") { - let c = s.trim_start_matches("color").parse::().unwrap_or_default(); + let c = s + .trim_start_matches("color") + .parse::() + .unwrap_or_default(); Some(Color::Indexed(c)) } else if s.contains("gray") { - let c = 232 + s.trim_start_matches("gray").parse::().unwrap_or_default(); + let c = 232 + + s.trim_start_matches("gray") + .parse::() + .unwrap_or_default(); Some(Color::Indexed(c)) } else if s.contains("rgb") { let red = (s.as_bytes()[3] as char).to_digit(10).unwrap_or_default() as u8; @@ -445,7 +470,11 @@ mod tests { fn test_config() -> Result<()> { let c = Config::new()?; assert_eq!( - c.keybindings.get(&Scene::Home).unwrap().get(&parse_key_sequence("").unwrap_or_default()).unwrap(), + c.keybindings + .get(&Scene::Home) + .unwrap() + .get(&parse_key_sequence("").unwrap_or_default()) + .unwrap(), &Action::Quit ); Ok(()) @@ -453,27 +482,48 @@ mod tests { #[test] fn test_simple_keys() { - assert_eq!(parse_key_event("a").unwrap(), KeyEvent::new(KeyCode::Char('a'), KeyModifiers::empty())); + assert_eq!( + parse_key_event("a").unwrap(), + KeyEvent::new(KeyCode::Char('a'), KeyModifiers::empty()) + ); - assert_eq!(parse_key_event("enter").unwrap(), KeyEvent::new(KeyCode::Enter, KeyModifiers::empty())); + assert_eq!( + parse_key_event("enter").unwrap(), + KeyEvent::new(KeyCode::Enter, KeyModifiers::empty()) + ); - assert_eq!(parse_key_event("esc").unwrap(), KeyEvent::new(KeyCode::Esc, KeyModifiers::empty())); + assert_eq!( + parse_key_event("esc").unwrap(), + KeyEvent::new(KeyCode::Esc, KeyModifiers::empty()) + ); } #[test] fn test_with_modifiers() { - assert_eq!(parse_key_event("ctrl-a").unwrap(), KeyEvent::new(KeyCode::Char('a'), KeyModifiers::CONTROL)); + assert_eq!( + parse_key_event("ctrl-a").unwrap(), + KeyEvent::new(KeyCode::Char('a'), KeyModifiers::CONTROL) + ); - assert_eq!(parse_key_event("alt-enter").unwrap(), KeyEvent::new(KeyCode::Enter, KeyModifiers::ALT)); + assert_eq!( + parse_key_event("alt-enter").unwrap(), + KeyEvent::new(KeyCode::Enter, KeyModifiers::ALT) + ); - assert_eq!(parse_key_event("shift-esc").unwrap(), KeyEvent::new(KeyCode::Esc, KeyModifiers::SHIFT)); + assert_eq!( + parse_key_event("shift-esc").unwrap(), + KeyEvent::new(KeyCode::Esc, KeyModifiers::SHIFT) + ); } #[test] fn test_multiple_modifiers() { assert_eq!( parse_key_event("ctrl-alt-a").unwrap(), - KeyEvent::new(KeyCode::Char('a'), KeyModifiers::CONTROL | KeyModifiers::ALT) + KeyEvent::new( + KeyCode::Char('a'), + KeyModifiers::CONTROL | KeyModifiers::ALT + ) ); assert_eq!( @@ -485,7 +535,10 @@ mod tests { #[test] fn test_reverse_multiple_modifiers() { assert_eq!( - key_event_to_string(&KeyEvent::new(KeyCode::Char('a'), KeyModifiers::CONTROL | KeyModifiers::ALT)), + key_event_to_string(&KeyEvent::new( + KeyCode::Char('a'), + KeyModifiers::CONTROL | KeyModifiers::ALT + )), "ctrl-alt-a".to_string() ); } @@ -498,8 +551,14 @@ mod tests { #[test] fn test_case_insensitivity() { - assert_eq!(parse_key_event("CTRL-a").unwrap(), KeyEvent::new(KeyCode::Char('a'), KeyModifiers::CONTROL)); + assert_eq!( + parse_key_event("CTRL-a").unwrap(), + KeyEvent::new(KeyCode::Char('a'), KeyModifiers::CONTROL) + ); - assert_eq!(parse_key_event("AlT-eNtEr").unwrap(), KeyEvent::new(KeyCode::Enter, KeyModifiers::ALT)); + assert_eq!( + parse_key_event("AlT-eNtEr").unwrap(), + KeyEvent::new(KeyCode::Enter, KeyModifiers::ALT) + ); } } diff --git a/node-launchpad/src/tui.rs b/node-launchpad/src/tui.rs index 4ddfb19a8c..3c4643e253 100644 --- a/node-launchpad/src/tui.rs +++ b/node-launchpad/src/tui.rs @@ -15,8 +15,8 @@ use color_eyre::eyre::Result; use crossterm::{ cursor, event::{ - DisableBracketedPaste, DisableMouseCapture, EnableBracketedPaste, EnableMouseCapture, Event as CrosstermEvent, - KeyEvent, KeyEventKind, MouseEvent, + DisableBracketedPaste, DisableMouseCapture, EnableBracketedPaste, EnableMouseCapture, + Event as CrosstermEvent, KeyEvent, KeyEventKind, MouseEvent, }, terminal::{EnterAlternateScreen, LeaveAlternateScreen}, }; @@ -73,7 +73,17 @@ impl Tui { let task = tokio::spawn(async {}); let mouse = false; let paste = false; - Ok(Self { terminal, task, cancellation_token, event_rx, event_tx, frame_rate, tick_rate, mouse, paste }) + Ok(Self { + terminal, + task, + cancellation_token, + event_rx, + event_tx, + frame_rate, + tick_rate, + mouse, + paste, + }) } pub fn tick_rate(mut self, tick_rate: f64) -> Self { diff --git a/node-launchpad/src/utils.rs b/node-launchpad/src/utils.rs index de891be601..f23e683cdf 100644 --- a/node-launchpad/src/utils.rs +++ b/node-launchpad/src/utils.rs @@ -13,17 +13,29 @@ use directories::ProjectDirs; use lazy_static::lazy_static; use tracing::error; use tracing_error::ErrorLayer; -use tracing_subscriber::{self, prelude::__tracing_subscriber_SubscriberExt, util::SubscriberInitExt, Layer}; - -const VERSION_MESSAGE: &str = - concat!(env!("CARGO_PKG_VERSION"), "-", env!("VERGEN_GIT_DESCRIBE"), " (", env!("VERGEN_BUILD_DATE"), ")"); +use tracing_subscriber::{ + self, prelude::__tracing_subscriber_SubscriberExt, util::SubscriberInitExt, Layer, +}; + +const VERSION_MESSAGE: &str = concat!( + env!("CARGO_PKG_VERSION"), + "-", + env!("VERGEN_GIT_DESCRIBE"), + " (", + env!("VERGEN_BUILD_DATE"), + ")" +); lazy_static! { pub static ref PROJECT_NAME: String = env!("CARGO_CRATE_NAME").to_uppercase().to_string(); pub static ref DATA_FOLDER: Option = - std::env::var(format!("{}_DATA", PROJECT_NAME.clone())).ok().map(PathBuf::from); + std::env::var(format!("{}_DATA", PROJECT_NAME.clone())) + .ok() + .map(PathBuf::from); pub static ref CONFIG_FOLDER: Option = - std::env::var(format!("{}_CONFIG", PROJECT_NAME.clone())).ok().map(PathBuf::from); + std::env::var(format!("{}_CONFIG", PROJECT_NAME.clone())) + .ok() + .map(PathBuf::from); pub static ref LOG_ENV: String = format!("{}_LOGLEVEL", PROJECT_NAME.clone()); pub static ref LOG_FILE: String = format!("{}.log", env!("CARGO_PKG_NAME")); } @@ -34,7 +46,10 @@ fn project_directory() -> Option { pub fn initialize_panic_handler() -> Result<()> { let (panic_hook, eyre_hook) = color_eyre::config::HookBuilder::default() - .panic_section(format!("This is a bug. Consider reporting it at {}", env!("CARGO_PKG_REPOSITORY"))) + .panic_section(format!( + "This is a bug. Consider reporting it at {}", + env!("CARGO_PKG_REPOSITORY") + )) .capture_span_trace_by_default(false) .display_location_section(false) .display_env_section(false) @@ -59,7 +74,8 @@ pub fn initialize_panic_handler() -> Result<()> { let file_path = handle_dump(&meta, panic_info); // prints human-panic message - print_msg(file_path, &meta).expect("human-panic: printing error message to console failed"); + print_msg(file_path, &meta) + .expect("human-panic: printing error message to console failed"); eprintln!("{}", panic_hook.panic_report(panic_info)); // prints color-eyre stack trace to stderr } let msg = format!("{}", panic_hook.panic_report(panic_info)); @@ -120,7 +136,10 @@ pub fn initialize_logging() -> Result<()> { .with_target(false) .with_ansi(false) .with_filter(tracing_subscriber::filter::EnvFilter::from_default_env()); - tracing_subscriber::registry().with(file_subscriber).with(ErrorLayer::default()).init(); + tracing_subscriber::registry() + .with(file_subscriber) + .with(ErrorLayer::default()) + .init(); Ok(()) } From 7c410fc1294a8186dfe5111fc144381032c41886 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 1 May 2024 17:51:13 +0100 Subject: [PATCH 191/205] chore: rename `node-launchpad` crate to `sn_node_launchpad` The binary name will be `node-launchpad`, but the crate is named consistently along with the other crates in the workspace (including the use of underscores rather than hyphens). --- Cargo.lock | 72 +++++++++---------- Cargo.toml | 2 +- .../.config/config.json5 | 0 {node-launchpad => sn_node_launchpad}/.envrc | 0 .../.gitignore | 0 .../Cargo.toml | 6 +- {node-launchpad => sn_node_launchpad}/LICENSE | 0 .../README.md | 0 .../build.rs | 0 .../src/action.rs | 0 .../src/app.rs | 0 .../src/cli.rs | 0 .../src/components.rs | 0 .../src/components/home.rs | 0 .../src/components/options.rs | 0 .../src/components/tab.rs | 0 .../src/config.rs | 0 .../src/main.rs | 0 .../src/mode.rs | 0 .../src/tui.rs | 0 .../src/utils.rs | 0 21 files changed, 42 insertions(+), 38 deletions(-) rename {node-launchpad => sn_node_launchpad}/.config/config.json5 (100%) rename {node-launchpad => sn_node_launchpad}/.envrc (100%) rename {node-launchpad => sn_node_launchpad}/.gitignore (100%) rename {node-launchpad => sn_node_launchpad}/Cargo.toml (96%) rename {node-launchpad => sn_node_launchpad}/LICENSE (100%) rename {node-launchpad => sn_node_launchpad}/README.md (100%) rename {node-launchpad => sn_node_launchpad}/build.rs (100%) rename {node-launchpad => sn_node_launchpad}/src/action.rs (100%) rename {node-launchpad => sn_node_launchpad}/src/app.rs (100%) rename {node-launchpad => sn_node_launchpad}/src/cli.rs (100%) rename {node-launchpad => sn_node_launchpad}/src/components.rs (100%) rename {node-launchpad => sn_node_launchpad}/src/components/home.rs (100%) rename {node-launchpad => sn_node_launchpad}/src/components/options.rs (100%) rename {node-launchpad => sn_node_launchpad}/src/components/tab.rs (100%) rename {node-launchpad => sn_node_launchpad}/src/config.rs (100%) rename {node-launchpad => sn_node_launchpad}/src/main.rs (100%) rename {node-launchpad => sn_node_launchpad}/src/mode.rs (100%) rename {node-launchpad => sn_node_launchpad}/src/tui.rs (100%) rename {node-launchpad => sn_node_launchpad}/src/utils.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index f98f31f7d1..528f4daed2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4585,42 +4585,6 @@ dependencies = [ "libc", ] -[[package]] -name = "node-launchpad" -version = "0.1.0" -dependencies = [ - "better-panic", - "clap", - "color-eyre", - "config", - "crossterm", - "derive_deref", - "directories", - "futures", - "human-panic", - "json5", - "lazy_static", - "libc", - "log", - "pretty_assertions", - "ratatui", - "serde", - "serde_json", - "signal-hook", - "sn-node-manager", - "sn_peers_acquisition", - "sn_service_management", - "strip-ansi-escapes", - "strum", - "tokio", - "tokio-util 0.7.10", - "tracing", - "tracing-error", - "tracing-subscriber", - "tui-input", - "vergen", -] - [[package]] name = "nohash-hasher" version = "0.2.0" @@ -7168,6 +7132,42 @@ dependencies = [ "xor_name", ] +[[package]] +name = "sn_node_launchpad" +version = "0.1.0" +dependencies = [ + "better-panic", + "clap", + "color-eyre", + "config", + "crossterm", + "derive_deref", + "directories", + "futures", + "human-panic", + "json5", + "lazy_static", + "libc", + "log", + "pretty_assertions", + "ratatui", + "serde", + "serde_json", + "signal-hook", + "sn-node-manager", + "sn_peers_acquisition", + "sn_service_management", + "strip-ansi-escapes", + "strum", + "tokio", + "tokio-util 0.7.10", + "tracing", + "tracing-error", + "tracing-subscriber", + "tui-input", + "vergen", +] + [[package]] name = "sn_node_rpc_client" version = "0.6.8" diff --git a/Cargo.toml b/Cargo.toml index 9998e149c7..9e7cccdbe2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,6 @@ [workspace] resolver = "2" members = [ - "node-launchpad", "sn_auditor", "sn_build_info", "sn_cli", @@ -11,6 +10,7 @@ members = [ "sn_metrics", "sn_networking", "sn_node", + "sn_node_launchpad", "sn_node_manager", "sn_node_rpc_client", "sn_peers_acquisition", diff --git a/node-launchpad/.config/config.json5 b/sn_node_launchpad/.config/config.json5 similarity index 100% rename from node-launchpad/.config/config.json5 rename to sn_node_launchpad/.config/config.json5 diff --git a/node-launchpad/.envrc b/sn_node_launchpad/.envrc similarity index 100% rename from node-launchpad/.envrc rename to sn_node_launchpad/.envrc diff --git a/node-launchpad/.gitignore b/sn_node_launchpad/.gitignore similarity index 100% rename from node-launchpad/.gitignore rename to sn_node_launchpad/.gitignore diff --git a/node-launchpad/Cargo.toml b/sn_node_launchpad/Cargo.toml similarity index 96% rename from node-launchpad/Cargo.toml rename to sn_node_launchpad/Cargo.toml index 139ccb9621..71073ffa90 100644 --- a/node-launchpad/Cargo.toml +++ b/sn_node_launchpad/Cargo.toml @@ -1,7 +1,7 @@ [package] authors = ["MaidSafe Developers "] description = "Node Launchpad" -name = "node-launchpad" +name = "sn_node_launchpad" version = "0.1.0" edition = "2021" license = "GPL-3.0" @@ -10,6 +10,10 @@ readme = "README.md" repository = "https://github.com/maidsafe/safe_network" build = "build.rs" +[[bin]] +name = "node-launchpad" +path = "src/main.rs" + [dependencies] sn_peers_acquisition = { verison = "0.2.10", path = "../sn_peers_acquisition", features = [ "network-contacts", diff --git a/node-launchpad/LICENSE b/sn_node_launchpad/LICENSE similarity index 100% rename from node-launchpad/LICENSE rename to sn_node_launchpad/LICENSE diff --git a/node-launchpad/README.md b/sn_node_launchpad/README.md similarity index 100% rename from node-launchpad/README.md rename to sn_node_launchpad/README.md diff --git a/node-launchpad/build.rs b/sn_node_launchpad/build.rs similarity index 100% rename from node-launchpad/build.rs rename to sn_node_launchpad/build.rs diff --git a/node-launchpad/src/action.rs b/sn_node_launchpad/src/action.rs similarity index 100% rename from node-launchpad/src/action.rs rename to sn_node_launchpad/src/action.rs diff --git a/node-launchpad/src/app.rs b/sn_node_launchpad/src/app.rs similarity index 100% rename from node-launchpad/src/app.rs rename to sn_node_launchpad/src/app.rs diff --git a/node-launchpad/src/cli.rs b/sn_node_launchpad/src/cli.rs similarity index 100% rename from node-launchpad/src/cli.rs rename to sn_node_launchpad/src/cli.rs diff --git a/node-launchpad/src/components.rs b/sn_node_launchpad/src/components.rs similarity index 100% rename from node-launchpad/src/components.rs rename to sn_node_launchpad/src/components.rs diff --git a/node-launchpad/src/components/home.rs b/sn_node_launchpad/src/components/home.rs similarity index 100% rename from node-launchpad/src/components/home.rs rename to sn_node_launchpad/src/components/home.rs diff --git a/node-launchpad/src/components/options.rs b/sn_node_launchpad/src/components/options.rs similarity index 100% rename from node-launchpad/src/components/options.rs rename to sn_node_launchpad/src/components/options.rs diff --git a/node-launchpad/src/components/tab.rs b/sn_node_launchpad/src/components/tab.rs similarity index 100% rename from node-launchpad/src/components/tab.rs rename to sn_node_launchpad/src/components/tab.rs diff --git a/node-launchpad/src/config.rs b/sn_node_launchpad/src/config.rs similarity index 100% rename from node-launchpad/src/config.rs rename to sn_node_launchpad/src/config.rs diff --git a/node-launchpad/src/main.rs b/sn_node_launchpad/src/main.rs similarity index 100% rename from node-launchpad/src/main.rs rename to sn_node_launchpad/src/main.rs diff --git a/node-launchpad/src/mode.rs b/sn_node_launchpad/src/mode.rs similarity index 100% rename from node-launchpad/src/mode.rs rename to sn_node_launchpad/src/mode.rs diff --git a/node-launchpad/src/tui.rs b/sn_node_launchpad/src/tui.rs similarity index 100% rename from node-launchpad/src/tui.rs rename to sn_node_launchpad/src/tui.rs diff --git a/node-launchpad/src/utils.rs b/sn_node_launchpad/src/utils.rs similarity index 100% rename from node-launchpad/src/utils.rs rename to sn_node_launchpad/src/utils.rs From 30c291f90418907e693f05a19a961307732d35f3 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 24 Apr 2024 21:22:47 +0100 Subject: [PATCH 192/205] fix: apply interval only to non-running nodes The node manager `start` command has an `--interval` argument that can be used as a time-based delay between starting nodes. The problem is, the interval would apply to nodes that were already running, which is a waste of time. Now the interval is not applied to those nodes. --- sn_node_manager/src/cmd/node.rs | 21 +++++++++++++++------ sn_service_management/src/lib.rs | 2 +- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/sn_node_manager/src/cmd/node.rs b/sn_node_manager/src/cmd/node.rs index 9df0fa4cea..778cfc1d5e 100644 --- a/sn_node_manager/src/cmd/node.rs +++ b/sn_node_manager/src/cmd/node.rs @@ -28,7 +28,7 @@ use sn_service_management::{ control::{ServiceControl, ServiceController}, get_local_node_registry_path, rpc::RpcClient, - NodeRegistry, NodeService, ServiceStatus, UpgradeOptions, UpgradeResult, + NodeRegistry, NodeService, ServiceStateActions, ServiceStatus, UpgradeOptions, UpgradeResult, }; use sn_transfers::HotWallet; use std::{io::Write, net::Ipv4Addr, path::PathBuf, str::FromStr}; @@ -271,8 +271,14 @@ pub async fn start( let service = NodeService::new(node, Box::new(rpc_client)); let mut service_manager = ServiceManager::new(service, Box::new(ServiceController {}), verbosity.clone()); - debug!("Sleeping for {} milliseconds", interval); - std::thread::sleep(std::time::Duration::from_millis(interval)); + if service_manager.service.status() != ServiceStatus::Running { + // It would be possible here to check if the service *is* running and then just + // continue without applying the delay. The reason for not doing so is because when + // `start` is called below, the user will get a message to say the service was already + // started, which I think is useful behaviour to retain. + debug!("Sleeping for {} milliseconds", interval); + std::thread::sleep(std::time::Duration::from_millis(interval)); + } match service_manager.start().await { Ok(()) => { node_registry.save()?; @@ -446,6 +452,12 @@ pub async fn upgrade( match service_manager.upgrade(options).await { Ok(upgrade_result) => { + if upgrade_result != UpgradeResult::NotRequired { + // It doesn't seem useful to apply the interval if there was no upgrade + // required for the previous service. + debug!("Sleeping for {} milliseconds", interval); + std::thread::sleep(std::time::Duration::from_millis(interval)); + } upgrade_summary.push(( service_manager.service.service_data.service_name.clone(), upgrade_result, @@ -458,9 +470,6 @@ pub async fn upgrade( )); } } - - debug!("Sleeping for {} milliseconds", interval); - std::thread::sleep(std::time::Duration::from_millis(interval)); } node_registry.save()?; diff --git a/sn_service_management/src/lib.rs b/sn_service_management/src/lib.rs index 9a2dca1f5e..93f6639c7e 100644 --- a/sn_service_management/src/lib.rs +++ b/sn_service_management/src/lib.rs @@ -44,7 +44,7 @@ pub enum ServiceStatus { Removed, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] pub enum UpgradeResult { Forced(String, String), NotRequired, From 42d71be6bc935bfbcc454761223803feb8874de9 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 24 Apr 2024 22:27:17 +0100 Subject: [PATCH 193/205] chore: use better banners These new banners for the node manager are a bit more eye catching, and also fix the incorrect sizes of the previous banners in the `status --details` command. --- sn_node_manager/src/cmd/node.rs | 40 +++++++++--------------------- sn_node_manager/src/lib.rs | 43 ++++++++++++++++++++++++++------- 2 files changed, 45 insertions(+), 38 deletions(-) diff --git a/sn_node_manager/src/cmd/node.rs b/sn_node_manager/src/cmd/node.rs index 778cfc1d5e..add83e2dd6 100644 --- a/sn_node_manager/src/cmd/node.rs +++ b/sn_node_manager/src/cmd/node.rs @@ -16,7 +16,7 @@ use crate::{ }, config, helpers::{download_and_extract_release, get_bin_version}, - refresh_node_registry, status_report, ServiceManager, VerbosityLevel, + print_banner, refresh_node_registry, status_report, ServiceManager, VerbosityLevel, }; use color_eyre::{eyre::eyre, Help, Result}; use colored::Colorize; @@ -57,9 +57,7 @@ pub async fn add( } if verbosity != VerbosityLevel::Minimal { - println!("================================================="); - println!(" Add Safenode Services "); - println!("================================================="); + print_banner("Add Safenode Services"); println!("{} service(s) to be added", count.unwrap_or(1)); } @@ -137,9 +135,7 @@ pub async fn balance( verbosity: VerbosityLevel, ) -> Result<()> { if verbosity != VerbosityLevel::Minimal { - println!("================================================="); - println!(" Reward Balances "); - println!("================================================="); + print_banner("Reward Balances"); } let mut node_registry = NodeRegistry::load(&config::get_node_registry_path()?)?; @@ -176,9 +172,7 @@ pub async fn remove( return Err(eyre!("The remove command must run as the root user")); } - println!("================================================="); - println!(" Remove Safenode Services "); - println!("================================================="); + print_banner("Remove Safenode Services"); let mut node_registry = NodeRegistry::load(&config::get_node_registry_path()?)?; refresh_node_registry(&mut node_registry, &ServiceController {}, true).await?; @@ -213,9 +207,7 @@ pub async fn reset(force: bool, verbosity: VerbosityLevel) -> Result<()> { return Err(eyre!("The reset command must run as the root user")); } - println!("================================================="); - println!(" Reset Safenode Services "); - println!("================================================="); + print_banner("Reset Safenode Services"); if !force { println!("WARNING: all safenode services, data, and logs will be removed."); @@ -249,9 +241,7 @@ pub async fn start( } if verbosity != VerbosityLevel::Minimal { - println!("================================================="); - println!(" Start Safenode Services "); - println!("================================================="); + print_banner("Start Safenode Services"); } let mut node_registry = NodeRegistry::load(&config::get_node_registry_path()?)?; @@ -294,9 +284,7 @@ pub async fn status(details: bool, fail: bool, json: bool) -> Result<()> { let mut local_node_registry = NodeRegistry::load(&get_local_node_registry_path()?)?; if !local_node_registry.nodes.is_empty() { if !json { - println!("================================================="); - println!(" Local Network "); - println!("================================================="); + print_banner("Local Network"); } status_report( &mut local_node_registry, @@ -312,10 +300,8 @@ pub async fn status(details: bool, fail: bool, json: bool) -> Result<()> { let mut node_registry = NodeRegistry::load(&config::get_node_registry_path()?)?; if !node_registry.nodes.is_empty() { - if !json { - println!("================================================="); - println!(" Safenode Services "); - println!("================================================="); + if !json && !details { + print_banner("Safenode Services"); } status_report( &mut node_registry, @@ -340,9 +326,7 @@ pub async fn stop( } if verbosity != VerbosityLevel::Minimal { - println!("================================================="); - println!(" Stop Safenode Services "); - println!("================================================="); + print_banner("Stop Safenode Services"); } let mut node_registry = NodeRegistry::load(&config::get_node_registry_path()?)?; @@ -395,9 +379,7 @@ pub async fn upgrade( let use_force = force || custom_bin_path.is_some(); if verbosity != VerbosityLevel::Minimal { - println!("================================================="); - println!(" Upgrade Safenode Services "); - println!("================================================="); + print_banner("Upgrade Safenode Services"); } let (upgrade_bin_path, target_version) = download_and_get_upgrade_bin_path( diff --git a/sn_node_manager/src/lib.rs b/sn_node_manager/src/lib.rs index f1e033efb5..f70bd28174 100644 --- a/sn_node_manager/src/lib.rs +++ b/sn_node_manager/src/lib.rs @@ -307,7 +307,11 @@ pub async fn status_report( println!("{json}"); } else if detailed_view { for node in &node_registry.nodes { - print_banner(&node.service_name, &node.status); + print_banner(&format!( + "{} - {}", + &node.service_name, + format_status_without_colour(&node.status) + )); println!("Version: {}", node.version); println!( "Peer ID: {}", @@ -334,13 +338,21 @@ pub async fn status_report( } if let Some(daemon) = &node_registry.daemon { - print_banner(&daemon.service_name, &daemon.status); + print_banner(&format!( + "{} - {}", + &daemon.service_name, + format_status(&daemon.status) + )); println!("Version: {}", daemon.version); println!("Bin path: {}", daemon.daemon_path.to_string_lossy()); } if let Some(faucet) = &node_registry.faucet { - print_banner(&faucet.service_name, &faucet.status); + print_banner(&format!( + "{} - {}", + &faucet.service_name, + format_status(&faucet.status) + )); println!("Version: {}", faucet.version); println!("Bin path: {}", faucet.faucet_path.to_string_lossy()); println!("Log path: {}", faucet.log_dir_path.to_string_lossy()); @@ -457,6 +469,18 @@ pub async fn refresh_node_registry( Ok(()) } +pub fn print_banner(text: &str) { + let padding = 2; + let text_width = text.len() + padding * 2; + let border_chars = 2; + let total_width = text_width + border_chars; + let top_bottom = "═".repeat(total_width); + + println!("╔{}╗", top_bottom); + println!("║ {:^width$} ║", text, width = text_width); + println!("╚{}╝", top_bottom); +} + fn format_status(status: &ServiceStatus) -> String { match status { ServiceStatus::Running => "RUNNING".green().to_string(), @@ -466,12 +490,13 @@ fn format_status(status: &ServiceStatus) -> String { } } -fn print_banner(service_name: &str, status: &ServiceStatus) { - let service_status = format!("{} - {}", service_name, format_status(status)); - let banner = "=".repeat(service_status.len()); - println!("{}", banner); - println!("{service_status}"); - println!("{}", banner); +fn format_status_without_colour(status: &ServiceStatus) -> String { + match status { + ServiceStatus::Running => "RUNNING".to_string(), + ServiceStatus::Stopped => "STOPPED".to_string(), + ServiceStatus::Added => "ADDED".to_string(), + ServiceStatus::Removed => "REMOVED".to_string(), + } } #[cfg(test)] From 924f7e89680301be24b9b352e87efcebd4fe49ca Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Fri, 26 Apr 2024 17:32:41 +0100 Subject: [PATCH 194/205] chore: output reward balance in `status --json` cmd This was a community feedback request. The reward balance is now output as part of the `status --json` command, which involved adding the field to the information tracked by the registry. BREAKING CHANGE: previously written registry files will not have this new field and so won't deserialize correctly. --- Cargo.lock | 1 + sn_node_manager/src/add_services/mod.rs | 2 ++ sn_node_manager/src/add_services/tests.rs | 3 +++ sn_node_manager/src/lib.rs | 33 ++++++++++++++++++----- sn_node_manager/src/local.rs | 21 ++++++++------- sn_node_manager/src/rpc.rs | 1 + sn_service_management/Cargo.toml | 1 + sn_service_management/src/node.rs | 2 ++ 8 files changed, 48 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 528f4daed2..0ba158c686 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7272,6 +7272,7 @@ dependencies = [ "serde_json", "service-manager", "sn_protocol", + "sn_transfers", "sysinfo", "thiserror", "tokio", diff --git a/sn_node_manager/src/add_services/mod.rs b/sn_node_manager/src/add_services/mod.rs index 9192124eb8..f9f3d79773 100644 --- a/sn_node_manager/src/add_services/mod.rs +++ b/sn_node_manager/src/add_services/mod.rs @@ -21,6 +21,7 @@ use sn_service_management::{ control::ServiceControl, DaemonServiceData, FaucetServiceData, NodeRegistry, NodeServiceData, ServiceStatus, }; +use sn_transfers::NanoTokens; use std::{ ffi::OsString, net::{IpAddr, Ipv4Addr, SocketAddr}, @@ -176,6 +177,7 @@ pub async fn add_node( local: options.local, log_dir_path: service_log_dir_path.clone(), number: node_number, + reward_balance: NanoTokens::zero(), rpc_socket_addr, peer_id: None, pid: None, diff --git a/sn_node_manager/src/add_services/tests.rs b/sn_node_manager/src/add_services/tests.rs index 20ccc28717..20e4ff07af 100644 --- a/sn_node_manager/src/add_services/tests.rs +++ b/sn_node_manager/src/add_services/tests.rs @@ -28,6 +28,7 @@ use sn_service_management::error::Result as ServiceControlResult; use sn_service_management::{ DaemonServiceData, FaucetServiceData, NodeRegistry, NodeServiceData, ServiceStatus, }; +use sn_transfers::NanoTokens; use std::{ ffi::OsString, net::{IpAddr, Ipv4Addr, SocketAddr}, @@ -207,6 +208,7 @@ async fn add_genesis_node_should_return_an_error_if_there_is_already_a_genesis_n number: 1, pid: None, peer_id: None, + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), status: ServiceStatus::Added, safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), @@ -785,6 +787,7 @@ async fn add_new_node_should_add_another_service() -> Result<()> { number: 1, pid: None, peer_id: None, + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), diff --git a/sn_node_manager/src/lib.rs b/sn_node_manager/src/lib.rs index f70bd28174..627f9107fc 100644 --- a/sn_node_manager/src/lib.rs +++ b/sn_node_manager/src/lib.rs @@ -332,8 +332,7 @@ pub async fn status_report( .as_ref() .map_or("-".to_string(), |p| p.len().to_string()) ); - let wallet = HotWallet::load_from(&node.data_dir_path)?; - println!("Reward balance: {}", wallet.balance()); + println!("Reward balance: {}", node.reward_balance); println!(); } @@ -424,6 +423,9 @@ pub async fn refresh_node_registry( } for node in &mut node_registry.nodes { + let wallet = HotWallet::load_from(&node.data_dir_path)?; + node.reward_balance = wallet.balance(); + let rpc_client = RpcClient::from_socket_addr(node.rpc_socket_addr); if let ServiceStatus::Running = node.status { if let Some(pid) = node.pid { @@ -515,6 +517,7 @@ mod tests { rpc::{NetworkInfo, NodeInfo, RecordAddress, RpcActions}, UpgradeOptions, UpgradeResult, }; + use sn_transfers::NanoTokens; use std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, path::{Path, PathBuf}, @@ -603,6 +606,7 @@ mod tests { number: 1, peer_id: None, pid: None, + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -689,6 +693,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: None, + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -744,6 +749,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: Some(1000), + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -835,6 +841,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: Some(1000), + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -903,6 +910,7 @@ mod tests { number: 1, peer_id: None, pid: None, + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -954,6 +962,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: Some(1000), + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -992,6 +1001,7 @@ mod tests { number: 1, peer_id: None, pid: None, + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -1032,6 +1042,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: None, + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -1071,6 +1082,7 @@ mod tests { number: 1, peer_id: None, pid: None, + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -1187,6 +1199,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: Some(1000), + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: current_node_bin.to_path_buf(), service_name: "safenode1".to_string(), @@ -1265,6 +1278,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: Some(1000), + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: current_node_bin.to_path_buf(), service_name: "safenode1".to_string(), @@ -1385,6 +1399,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: Some(1000), + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: current_node_bin.to_path_buf(), service_name: "safenode1".to_string(), @@ -1518,6 +1533,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: Some(1000), + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: current_node_bin.to_path_buf(), service_name: "safenode1".to_string(), @@ -1647,6 +1663,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: Some(1000), + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: current_node_bin.to_path_buf(), service_name: "safenode1".to_string(), @@ -1714,6 +1731,7 @@ mod tests { number: 1, pid: None, peer_id: None, + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: safenode_bin.to_path_buf(), status: ServiceStatus::Stopped, @@ -1762,6 +1780,7 @@ mod tests { peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -1812,14 +1831,15 @@ mod tests { local: false, log_dir_path: PathBuf::from("/var/log/safenode/safenode1"), number: 1, - rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), - safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), - service_name: "safenode1".to_string(), - status: ServiceStatus::Running, pid: Some(1000), peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), + reward_balance: NanoTokens::zero(), + rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), + safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), + service_name: "safenode1".to_string(), + status: ServiceStatus::Running, user: "safe".to_string(), version: "0.98.1".to_string(), }; @@ -1870,6 +1890,7 @@ mod tests { number: 1, pid: None, peer_id: None, + reward_balance: NanoTokens::zero(), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: safenode_bin.to_path_buf(), service_name: "safenode1".to_string(), diff --git a/sn_node_manager/src/local.rs b/sn_node_manager/src/local.rs index f127f2c728..335eace3d5 100644 --- a/sn_node_manager/src/local.rs +++ b/sn_node_manager/src/local.rs @@ -18,7 +18,7 @@ use sn_service_management::{ rpc::{RpcActions, RpcClient}, FaucetServiceData, NodeRegistry, NodeServiceData, ServiceStatus, }; -use sn_transfers::get_faucet_data_dir; +use sn_transfers::{get_faucet_data_dir, NanoTokens}; use std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, path::PathBuf, @@ -338,21 +338,22 @@ pub async fn run_node( Ok(NodeServiceData { connected_peers, + data_dir_path: node_info.data_path, genesis: run_options.genesis, home_network: false, + listen_addr: Some(listen_addrs), local: true, - service_name: format!("safenode-local{}", run_options.number), - user, + log_dir_path: node_info.log_path, number: run_options.number, - rpc_socket_addr: run_options.rpc_socket_addr, - version: run_options.version.to_string(), - status: ServiceStatus::Running, - pid: Some(node_info.pid), - listen_addr: Some(listen_addrs), peer_id: Some(peer_id), - log_dir_path: node_info.log_path, - data_dir_path: node_info.data_path, + pid: Some(node_info.pid), + reward_balance: NanoTokens::zero(), + rpc_socket_addr: run_options.rpc_socket_addr, safenode_path: launcher.get_safenode_path(), + status: ServiceStatus::Running, + service_name: format!("safenode-local{}", run_options.number), + user: get_username()?, + version: run_options.version.to_string(), }) } diff --git a/sn_node_manager/src/rpc.rs b/sn_node_manager/src/rpc.rs index fa558ece66..8d5e87a64f 100644 --- a/sn_node_manager/src/rpc.rs +++ b/sn_node_manager/src/rpc.rs @@ -167,6 +167,7 @@ pub async fn restart_node_service( number: new_node_number as u16, peer_id: None, pid: None, + reward_balance: current_node_clone.reward_balance, rpc_socket_addr: current_node_clone.rpc_socket_addr, safenode_path, service_name: new_service_name.clone(), diff --git a/sn_service_management/Cargo.toml b/sn_service_management/Cargo.toml index d8c6d274c8..01f71b9bdf 100644 --- a/sn_service_management/Cargo.toml +++ b/sn_service_management/Cargo.toml @@ -20,6 +20,7 @@ serde_json = "1.0" semver = "1.0.20" service-manager = "0.6.0" sn_protocol = { path = "../sn_protocol", version = "0.16.3", features = ["rpc"] } +sn_transfers = { path = "../sn_transfers", version = "0.17.2" } sysinfo = "0.30.8" thiserror = "1.0.23" tokio = { version = "1.32.0", features = ["time"] } diff --git a/sn_service_management/src/node.rs b/sn_service_management/src/node.rs index 6fdadb68e7..913c0b8cf8 100644 --- a/sn_service_management/src/node.rs +++ b/sn_service_management/src/node.rs @@ -12,6 +12,7 @@ use libp2p::{multiaddr::Protocol, Multiaddr, PeerId}; use serde::{de::Error as DeError, Deserialize, Deserializer, Serialize, Serializer}; use service_manager::{ServiceInstallCtx, ServiceLabel}; use sn_protocol::get_port_from_multiaddr; +use sn_transfers::NanoTokens; use std::{ffi::OsString, net::SocketAddr, path::PathBuf, str::FromStr}; pub struct NodeService<'a> { @@ -162,6 +163,7 @@ pub struct NodeServiceData { )] pub peer_id: Option, pub pid: Option, + pub reward_balance: NanoTokens, pub rpc_socket_addr: SocketAddr, pub safenode_path: PathBuf, pub service_name: String, From 50321d4b286c3d220339da33d8f2e07285c0e63a Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Tue, 30 Apr 2024 17:07:02 +0100 Subject: [PATCH 195/205] test: use node registry for status Rather than parsing the text-based output of the `status` command, we now switch to using `status --json`, which we can then serialize into `StatusSummary`, and any status can be obtained from there. This prevents breakage when the text output changes. --- sn_node_manager/tests/e2e.rs | 232 ++++++++++++------------------- sn_service_management/src/lib.rs | 6 +- 2 files changed, 93 insertions(+), 145 deletions(-) diff --git a/sn_node_manager/tests/e2e.rs b/sn_node_manager/tests/e2e.rs index 5780a7b1a5..017289243c 100644 --- a/sn_node_manager/tests/e2e.rs +++ b/sn_node_manager/tests/e2e.rs @@ -8,7 +8,7 @@ use assert_cmd::Command; use libp2p_identity::PeerId; -use std::str::FromStr; +use sn_service_management::{ServiceStatus, StatusSummary}; /// These tests need to execute as the root user. /// @@ -19,13 +19,6 @@ use std::str::FromStr; const CI_USER: &str = "runner"; -#[derive(Debug)] -struct ServiceStatus { - name: String, - peer_id: String, - status: String, -} - /// The default behaviour is for the service to run as the `safe` user, which gets created during /// the process. However, there seems to be some sort of issue with adding user accounts on the GHA /// build agent, so we will just tell it to use the `runner` user, which is the account for the @@ -47,198 +40,149 @@ fn cross_platform_service_install_and_control() { .assert() .success(); - let output = Command::cargo_bin("safenode-manager") - .unwrap() - .arg("status") - .output() - .expect("Could not retrieve service status"); - let service_status = parse_service_status(&output.stdout); - - assert_eq!(service_status[0].name, "safenode1"); - assert_eq!(service_status[0].peer_id, "-"); - assert_eq!(service_status[0].status, "ADDED"); - assert_eq!(service_status[1].name, "safenode2"); - assert_eq!(service_status[1].peer_id, "-"); - assert_eq!(service_status[1].status, "ADDED"); - assert_eq!(service_status[2].name, "safenode3"); - assert_eq!(service_status[2].peer_id, "-"); - assert_eq!(service_status[2].status, "ADDED"); + let registry = get_status(); + assert_eq!(registry.nodes[0].service_name, "safenode1"); + assert_eq!(registry.nodes[0].peer_id, None); + assert_eq!(registry.nodes[0].status, ServiceStatus::Added); + assert_eq!(registry.nodes[1].service_name, "safenode2"); + assert_eq!(registry.nodes[1].peer_id, None); + assert_eq!(registry.nodes[1].status, ServiceStatus::Added); + assert_eq!(registry.nodes[2].service_name, "safenode3"); + assert_eq!(registry.nodes[2].peer_id, None); + assert_eq!(registry.nodes[2].status, ServiceStatus::Added); // Start each of the three services. let mut cmd = Command::cargo_bin("safenode-manager").unwrap(); cmd.arg("start").assert().success(); - let output = Command::cargo_bin("safenode-manager") - .unwrap() - .arg("status") - .output() - .expect("Could not retrieve service status"); - let start_status = parse_service_status(&output.stdout); // After `start`, all services should be running with valid peer IDs assigned. - assert_eq!(start_status[0].name, "safenode1"); - assert_eq!(start_status[0].status, "RUNNING"); - assert_eq!(start_status[1].name, "safenode2"); - assert_eq!(start_status[1].status, "RUNNING"); - assert_eq!(start_status[2].name, "safenode3"); - assert_eq!(start_status[2].status, "RUNNING"); - for status in start_status.iter() { - assert!(PeerId::from_str(&status.peer_id).is_ok()); - } + let registry = get_status(); + assert_eq!(registry.nodes[0].service_name, "safenode1"); + assert_eq!(registry.nodes[0].status, ServiceStatus::Running); + assert_eq!(registry.nodes[1].service_name, "safenode2"); + assert_eq!(registry.nodes[1].status, ServiceStatus::Running); + assert_eq!(registry.nodes[2].service_name, "safenode3"); + assert_eq!(registry.nodes[2].status, ServiceStatus::Running); // The three peer IDs should persist throughout the rest of the test. - let peer_ids = start_status + let peer_ids = registry + .nodes .iter() - .map(|s| s.peer_id.clone()) - .collect::>(); + .map(|n| n.peer_id) + .collect::>>(); // Stop each of the three services. let mut cmd = Command::cargo_bin("safenode-manager").unwrap(); cmd.arg("stop").assert().success(); - let output = Command::cargo_bin("safenode-manager") - .unwrap() - .arg("status") - .output() - .expect("Could not retrieve service status"); - let stop_status = parse_service_status(&output.stdout); // After `stop`, all services should be stopped with peer IDs retained. - assert_eq!(stop_status[0].name, "safenode1"); - assert_eq!(stop_status[0].status, "STOPPED"); - assert_eq!(stop_status[0].peer_id, peer_ids[0]); - assert_eq!(stop_status[1].name, "safenode2"); - assert_eq!(stop_status[1].status, "STOPPED"); - assert_eq!(stop_status[1].peer_id, peer_ids[1]); - assert_eq!(stop_status[2].name, "safenode3"); - assert_eq!(stop_status[2].status, "STOPPED"); - assert_eq!(stop_status[2].peer_id, peer_ids[2]); + let registry = get_status(); + assert_eq!(registry.nodes[0].service_name, "safenode1"); + assert_eq!(registry.nodes[0].status, ServiceStatus::Stopped); + assert_eq!(registry.nodes[0].peer_id, peer_ids[0]); + assert_eq!(registry.nodes[1].service_name, "safenode2"); + assert_eq!(registry.nodes[1].status, ServiceStatus::Stopped); + assert_eq!(registry.nodes[1].peer_id, peer_ids[1]); + assert_eq!(registry.nodes[2].service_name, "safenode3"); + assert_eq!(registry.nodes[2].status, ServiceStatus::Stopped); + assert_eq!(registry.nodes[2].peer_id, peer_ids[2]); // Start each of the three services again. let mut cmd = Command::cargo_bin("safenode-manager").unwrap(); cmd.arg("start").assert().success(); - let output = Command::cargo_bin("safenode-manager") - .unwrap() - .arg("status") - .output() - .expect("Could not retrieve service status"); - let start_status = parse_service_status(&output.stdout); // Peer IDs again should be retained after restart. - assert_eq!(start_status[0].name, "safenode1"); - assert_eq!(start_status[0].status, "RUNNING"); - assert_eq!(start_status[0].peer_id, peer_ids[0]); - assert_eq!(start_status[1].name, "safenode2"); - assert_eq!(start_status[1].status, "RUNNING"); - assert_eq!(start_status[1].peer_id, peer_ids[1]); - assert_eq!(start_status[2].name, "safenode3"); - assert_eq!(start_status[2].status, "RUNNING"); - assert_eq!(start_status[2].peer_id, peer_ids[2]); + let registry = get_status(); + assert_eq!(registry.nodes[0].service_name, "safenode1"); + assert_eq!(registry.nodes[0].status, ServiceStatus::Running); + assert_eq!(registry.nodes[0].peer_id, peer_ids[0]); + assert_eq!(registry.nodes[1].service_name, "safenode2"); + assert_eq!(registry.nodes[1].status, ServiceStatus::Running); + assert_eq!(registry.nodes[1].peer_id, peer_ids[1]); + assert_eq!(registry.nodes[2].service_name, "safenode3"); + assert_eq!(registry.nodes[2].status, ServiceStatus::Running); + assert_eq!(registry.nodes[2].peer_id, peer_ids[2]); // Stop two nodes by peer ID. let mut cmd = Command::cargo_bin("safenode-manager").unwrap(); cmd.arg("stop") .arg("--peer-id") - .arg(start_status[0].peer_id.clone()) + .arg(registry.nodes[0].peer_id.unwrap().to_string()) .arg("--peer-id") - .arg(start_status[2].peer_id.clone()) + .arg(registry.nodes[2].peer_id.unwrap().to_string()) .assert() .success(); - let output = Command::cargo_bin("safenode-manager") - .unwrap() - .arg("status") - .output() - .expect("Could not retrieve service status"); - let stop_status = parse_service_status(&output.stdout); // Peer IDs again should be retained after restart. - assert_eq!(stop_status[0].name, "safenode1"); - assert_eq!(stop_status[0].status, "STOPPED"); - assert_eq!(stop_status[0].peer_id, peer_ids[0]); - assert_eq!(stop_status[1].name, "safenode2"); - assert_eq!(stop_status[1].status, "RUNNING"); - assert_eq!(stop_status[1].peer_id, peer_ids[1]); - assert_eq!(stop_status[2].name, "safenode3"); - assert_eq!(stop_status[2].status, "STOPPED"); - assert_eq!(stop_status[2].peer_id, peer_ids[2]); + let registry = get_status(); + assert_eq!(registry.nodes[0].service_name, "safenode1"); + assert_eq!(registry.nodes[0].status, ServiceStatus::Stopped); + assert_eq!(registry.nodes[0].peer_id, peer_ids[0]); + assert_eq!(registry.nodes[1].service_name, "safenode2"); + assert_eq!(registry.nodes[1].status, ServiceStatus::Running); + assert_eq!(registry.nodes[1].peer_id, peer_ids[1]); + assert_eq!(registry.nodes[2].service_name, "safenode3"); + assert_eq!(registry.nodes[2].status, ServiceStatus::Stopped); + assert_eq!(registry.nodes[2].peer_id, peer_ids[2]); // Now restart the stopped nodes by service name. let mut cmd = Command::cargo_bin("safenode-manager").unwrap(); cmd.arg("start") .arg("--service-name") - .arg(stop_status[0].name.clone()) + .arg(registry.nodes[0].service_name.clone()) .arg("--service-name") - .arg(stop_status[2].name.clone()) + .arg(registry.nodes[2].service_name.clone()) .assert() .success(); - let output = Command::cargo_bin("safenode-manager") - .unwrap() - .arg("status") - .output() - .expect("Could not retrieve service status"); - let single_node_start_status = parse_service_status(&output.stdout); // The stopped nodes should now be running again. - assert_eq!(single_node_start_status[0].name, "safenode1"); - assert_eq!(single_node_start_status[0].status, "RUNNING"); - assert_eq!(single_node_start_status[0].peer_id, peer_ids[0]); - assert_eq!(single_node_start_status[1].name, "safenode2"); - assert_eq!(single_node_start_status[1].status, "RUNNING"); - assert_eq!(single_node_start_status[1].peer_id, peer_ids[1]); - assert_eq!(single_node_start_status[2].name, "safenode3"); - assert_eq!(single_node_start_status[2].status, "RUNNING"); - assert_eq!(single_node_start_status[2].peer_id, peer_ids[2]); + let registry = get_status(); + assert_eq!(registry.nodes[0].service_name, "safenode1"); + assert_eq!(registry.nodes[0].status, ServiceStatus::Running); + assert_eq!(registry.nodes[0].peer_id, peer_ids[0]); + assert_eq!(registry.nodes[1].service_name, "safenode2"); + assert_eq!(registry.nodes[1].status, ServiceStatus::Running); + assert_eq!(registry.nodes[1].peer_id, peer_ids[1]); + assert_eq!(registry.nodes[2].service_name, "safenode3"); + assert_eq!(registry.nodes[2].status, ServiceStatus::Running); + assert_eq!(registry.nodes[2].peer_id, peer_ids[2]); // Finally, stop each of the three services. let mut cmd = Command::cargo_bin("safenode-manager").unwrap(); cmd.arg("stop").assert().success(); - let output = Command::cargo_bin("safenode-manager") - .unwrap() - .arg("status") - .output() - .expect("Could not retrieve service status"); - let stop_status = parse_service_status(&output.stdout); // After `stop`, all services should be stopped with peer IDs retained. - assert_eq!(stop_status[0].name, "safenode1"); - assert_eq!(stop_status[0].status, "STOPPED"); - assert_eq!(stop_status[0].peer_id, peer_ids[0]); - assert_eq!(stop_status[1].name, "safenode2"); - assert_eq!(stop_status[1].status, "STOPPED"); - assert_eq!(stop_status[1].peer_id, peer_ids[1]); - assert_eq!(stop_status[2].name, "safenode3"); - assert_eq!(stop_status[2].status, "STOPPED"); - assert_eq!(stop_status[2].peer_id, peer_ids[2]); - - // Remove a two nodes. + let registry = get_status(); + assert_eq!(registry.nodes[0].service_name, "safenode1"); + assert_eq!(registry.nodes[0].status, ServiceStatus::Stopped); + assert_eq!(registry.nodes[0].peer_id, peer_ids[0]); + assert_eq!(registry.nodes[1].service_name, "safenode2"); + assert_eq!(registry.nodes[1].status, ServiceStatus::Stopped); + assert_eq!(registry.nodes[1].peer_id, peer_ids[1]); + assert_eq!(registry.nodes[2].service_name, "safenode3"); + assert_eq!(registry.nodes[2].status, ServiceStatus::Stopped); + assert_eq!(registry.nodes[2].peer_id, peer_ids[2]); + + // Remove two nodes. let mut cmd = Command::cargo_bin("safenode-manager").unwrap(); cmd.arg("remove") .arg("--service-name") - .arg(stop_status[0].name.clone()) + .arg(registry.nodes[0].service_name.clone()) .arg("--service-name") - .arg(stop_status[1].name.clone()) + .arg(registry.nodes[1].service_name.clone()) .assert() .success(); + let registry = get_status(); + assert_eq!(registry.nodes.len(), 1); +} + +fn get_status() -> StatusSummary { let output = Command::cargo_bin("safenode-manager") .unwrap() .arg("status") + .arg("--json") .output() .expect("Could not retrieve service status"); - let remove_status = parse_service_status(&output.stdout); - assert_eq!(remove_status.len(), 1); -} - -fn parse_service_status(output: &[u8]) -> Vec { - let output_str = String::from_utf8_lossy(output); - output_str - .split('\n') - .skip(5) // Skip header lines - .filter(|line| !line.is_empty()) - .map(|line| { - let columns: Vec<&str> = line.split_whitespace().collect(); - ServiceStatus { - name: columns[0].to_string(), - peer_id: columns[1].to_string(), - status: columns[2].to_string(), - } - }) - .collect() + let output = String::from_utf8_lossy(&output.stdout).to_string(); + serde_json::from_str(&output).unwrap() } diff --git a/sn_service_management/src/lib.rs b/sn_service_management/src/lib.rs index 93f6639c7e..5af30c90fe 100644 --- a/sn_service_management/src/lib.rs +++ b/sn_service_management/src/lib.rs @@ -138,7 +138,11 @@ impl NodeRegistry { }); } - let registry = serde_json::from_str(&contents)?; + Self::from_json(&contents) + } + + pub fn from_json(json: &str) -> Result { + let registry = serde_json::from_str(json)?; Ok(registry) } From 37168903f4e8eb8ad20efbd191794124dc92ba35 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Tue, 30 Apr 2024 19:19:22 +0100 Subject: [PATCH 196/205] fix: change reward balance to optional Due to it being possible to run the node manager status command before any services have been started, the reward balance should be an optional value, since the node's wallet has not been created before it starts. --- sn_node_manager/src/add_services/mod.rs | 3 +- sn_node_manager/src/add_services/tests.rs | 4 +- sn_node_manager/src/lib.rs | 50 +++++++++++++---------- sn_node_manager/src/local.rs | 4 +- sn_service_management/src/node.rs | 2 +- 5 files changed, 35 insertions(+), 28 deletions(-) diff --git a/sn_node_manager/src/add_services/mod.rs b/sn_node_manager/src/add_services/mod.rs index f9f3d79773..39e0b110dd 100644 --- a/sn_node_manager/src/add_services/mod.rs +++ b/sn_node_manager/src/add_services/mod.rs @@ -21,7 +21,6 @@ use sn_service_management::{ control::ServiceControl, DaemonServiceData, FaucetServiceData, NodeRegistry, NodeServiceData, ServiceStatus, }; -use sn_transfers::NanoTokens; use std::{ ffi::OsString, net::{IpAddr, Ipv4Addr, SocketAddr}, @@ -177,7 +176,7 @@ pub async fn add_node( local: options.local, log_dir_path: service_log_dir_path.clone(), number: node_number, - reward_balance: NanoTokens::zero(), + reward_balance: None, rpc_socket_addr, peer_id: None, pid: None, diff --git a/sn_node_manager/src/add_services/tests.rs b/sn_node_manager/src/add_services/tests.rs index 20e4ff07af..55db7cb7f0 100644 --- a/sn_node_manager/src/add_services/tests.rs +++ b/sn_node_manager/src/add_services/tests.rs @@ -208,7 +208,7 @@ async fn add_genesis_node_should_return_an_error_if_there_is_already_a_genesis_n number: 1, pid: None, peer_id: None, - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), status: ServiceStatus::Added, safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), @@ -787,7 +787,7 @@ async fn add_new_node_should_add_another_service() -> Result<()> { number: 1, pid: None, peer_id: None, - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), diff --git a/sn_node_manager/src/lib.rs b/sn_node_manager/src/lib.rs index 627f9107fc..a3a01320f6 100644 --- a/sn_node_manager/src/lib.rs +++ b/sn_node_manager/src/lib.rs @@ -332,7 +332,11 @@ pub async fn status_report( .as_ref() .map_or("-".to_string(), |p| p.len().to_string()) ); - println!("Reward balance: {}", node.reward_balance); + println!( + "Reward balance: {}", + node.reward_balance + .map_or("-".to_string(), |b| b.to_string()) + ); println!(); } @@ -423,8 +427,12 @@ pub async fn refresh_node_registry( } for node in &mut node_registry.nodes { - let wallet = HotWallet::load_from(&node.data_dir_path)?; - node.reward_balance = wallet.balance(); + // The `status` command can run before a node is started and therefore before its wallet + // exists. + match HotWallet::load_from(&node.data_dir_path) { + Ok(wallet) => node.reward_balance = Some(wallet.balance()), + Err(_) => node.reward_balance = None, + } let rpc_client = RpcClient::from_socket_addr(node.rpc_socket_addr); if let ServiceStatus::Running = node.status { @@ -606,7 +614,7 @@ mod tests { number: 1, peer_id: None, pid: None, - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -693,7 +701,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: None, - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -749,7 +757,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: Some(1000), - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -841,7 +849,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: Some(1000), - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -910,7 +918,7 @@ mod tests { number: 1, peer_id: None, pid: None, - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -962,7 +970,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: Some(1000), - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -1001,7 +1009,7 @@ mod tests { number: 1, peer_id: None, pid: None, - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -1042,7 +1050,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: None, - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -1082,7 +1090,7 @@ mod tests { number: 1, peer_id: None, pid: None, - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -1199,7 +1207,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: Some(1000), - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: current_node_bin.to_path_buf(), service_name: "safenode1".to_string(), @@ -1278,7 +1286,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: Some(1000), - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: current_node_bin.to_path_buf(), service_name: "safenode1".to_string(), @@ -1399,7 +1407,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: Some(1000), - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: current_node_bin.to_path_buf(), service_name: "safenode1".to_string(), @@ -1533,7 +1541,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: Some(1000), - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: current_node_bin.to_path_buf(), service_name: "safenode1".to_string(), @@ -1663,7 +1671,7 @@ mod tests { "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), pid: Some(1000), - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: current_node_bin.to_path_buf(), service_name: "safenode1".to_string(), @@ -1731,7 +1739,7 @@ mod tests { number: 1, pid: None, peer_id: None, - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: safenode_bin.to_path_buf(), status: ServiceStatus::Stopped, @@ -1780,7 +1788,7 @@ mod tests { peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -1835,7 +1843,7 @@ mod tests { peer_id: Some(PeerId::from_str( "12D3KooWS2tpXGGTmg2AHFiDh57yPQnat49YHnyqoggzXZWpqkCR", )?), - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: PathBuf::from("/var/safenode-manager/services/safenode1/safenode"), service_name: "safenode1".to_string(), @@ -1890,7 +1898,7 @@ mod tests { number: 1, pid: None, peer_id: None, - reward_balance: NanoTokens::zero(), + reward_balance: Some(NanoTokens::zero()), rpc_socket_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8081), safenode_path: safenode_bin.to_path_buf(), service_name: "safenode1".to_string(), diff --git a/sn_node_manager/src/local.rs b/sn_node_manager/src/local.rs index 335eace3d5..685685cffe 100644 --- a/sn_node_manager/src/local.rs +++ b/sn_node_manager/src/local.rs @@ -18,7 +18,7 @@ use sn_service_management::{ rpc::{RpcActions, RpcClient}, FaucetServiceData, NodeRegistry, NodeServiceData, ServiceStatus, }; -use sn_transfers::{get_faucet_data_dir, NanoTokens}; +use sn_transfers::get_faucet_data_dir; use std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, path::PathBuf, @@ -347,7 +347,7 @@ pub async fn run_node( number: run_options.number, peer_id: Some(peer_id), pid: Some(node_info.pid), - reward_balance: NanoTokens::zero(), + reward_balance: None, rpc_socket_addr: run_options.rpc_socket_addr, safenode_path: launcher.get_safenode_path(), status: ServiceStatus::Running, diff --git a/sn_service_management/src/node.rs b/sn_service_management/src/node.rs index 913c0b8cf8..a4e789d662 100644 --- a/sn_service_management/src/node.rs +++ b/sn_service_management/src/node.rs @@ -163,7 +163,7 @@ pub struct NodeServiceData { )] pub peer_id: Option, pub pid: Option, - pub reward_balance: NanoTokens, + pub reward_balance: Option, pub rpc_socket_addr: SocketAddr, pub safenode_path: PathBuf, pub service_name: String, From 79ea950913c89f7cc6f6ff92fd2d9d360e760e6a Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Tue, 30 Apr 2024 19:59:21 +0100 Subject: [PATCH 197/205] test: disable node man integration tests These tests are no longer passing and it's not completely clear why. In any case, I think they need to be a bit more isolated, and operate on their own local network. At the moment there is not enough time to address that. --- .github/workflows/merge.yml | 345 ++++++++++++++++++------------------ 1 file changed, 174 insertions(+), 171 deletions(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 4039643392..4ee4c069e7 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -160,179 +160,182 @@ jobs: - shell: bash run: cargo test --lib --package sn-node-manager - node-manager-e2e-tests: - name: node manager e2e tests - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - - { os: ubuntu-latest, elevated: sudo env PATH="$PATH" } - - { os: macos-latest, elevated: sudo } - - { os: windows-latest } - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - - shell: bash - if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' - run: | - ${{ matrix.elevated }} rustup default stable - ${{ matrix.elevated }} cargo test --package sn-node-manager --release --test e2e -- --nocapture - - # Powershell step runs as admin by default. - - name: run integration test in powershell - if: matrix.os == 'windows-latest' - shell: pwsh - run: | - curl -L -o WinSW.exe $env:WINSW_URL - - New-Item -ItemType Directory -Force -Path "$env:GITHUB_WORKSPACE\bin" - Move-Item -Path WinSW.exe -Destination "$env:GITHUB_WORKSPACE\bin" - $env:PATH += ";$env:GITHUB_WORKSPACE\bin" - - cargo test --release --package sn-node-manager --test e2e -- --nocapture +# +# Temporarily disable node manager integration tests until they can be made more isolated. +# + # node-manager-e2e-tests: + # name: node manager e2e tests + # runs-on: ${{ matrix.os }} + # strategy: + # fail-fast: false + # matrix: + # include: + # - { os: ubuntu-latest, elevated: sudo env PATH="$PATH" } + # - { os: macos-latest, elevated: sudo } + # - { os: windows-latest } + # steps: + # - uses: actions/checkout@v4 + # + # - name: Install Rust + # uses: dtolnay/rust-toolchain@stable + # - uses: Swatinem/rust-cache@v2 + # + # - shell: bash + # if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' + # run: | + # ${{ matrix.elevated }} rustup default stable + # ${{ matrix.elevated }} cargo test --package sn-node-manager --release --test e2e -- --nocapture + # + # # Powershell step runs as admin by default. + # - name: run integration test in powershell + # if: matrix.os == 'windows-latest' + # shell: pwsh + # run: | + # curl -L -o WinSW.exe $env:WINSW_URL + # + # New-Item -ItemType Directory -Force -Path "$env:GITHUB_WORKSPACE\bin" + # Move-Item -Path WinSW.exe -Destination "$env:GITHUB_WORKSPACE\bin" + # $env:PATH += ";$env:GITHUB_WORKSPACE\bin" + # + # cargo test --release --package sn-node-manager --test e2e -- --nocapture # Each upgrade test needs its own VM, otherwise they will interfere with each other. - node-manager-upgrade-tests: - name: node manager upgrade tests - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - - { - os: ubuntu-latest, - elevated: sudo env PATH="$PATH", - test: upgrade_to_latest_version, - } - - { - os: ubuntu-latest, - elevated: sudo env PATH="$PATH", - test: force_upgrade_when_two_binaries_have_the_same_version, - } - - { - os: ubuntu-latest, - elevated: sudo env PATH="$PATH", - test: force_downgrade_to_a_previous_version, - } - - { - os: ubuntu-latest, - elevated: sudo env PATH="$PATH", - test: upgrade_from_older_version_to_specific_version, - } - - { - os: macos-latest, - elevated: sudo, - test: upgrade_to_latest_version, - } - - { - os: macos-latest, - elevated: sudo, - test: force_upgrade_when_two_binaries_have_the_same_version, - } - - { - os: macos-latest, - elevated: sudo, - test: force_downgrade_to_a_previous_version, - } - - { - os: macos-latest, - elevated: sudo, - test: upgrade_from_older_version_to_specific_version, - } - - { os: windows-latest, test: upgrade_to_latest_version } - - { - os: windows-latest, - test: force_upgrade_when_two_binaries_have_the_same_version, - } - - { os: windows-latest, test: force_downgrade_to_a_previous_version } - - { - os: windows-latest, - test: upgrade_from_older_version_to_specific_version, - } - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - - shell: bash - if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' - run: | - ${{ matrix.elevated }} rustup default stable - ${{ matrix.elevated }} cargo test --package sn-node-manager --release \ - --test upgrades ${{ matrix.test }} -- --nocapture - - # Powershell step runs as admin by default. - - name: run integration test in powershell - if: matrix.os == 'windows-latest' - shell: pwsh - run: | - curl -L -o WinSW.exe $env:WINSW_URL - - New-Item -ItemType Directory -Force -Path "$env:GITHUB_WORKSPACE\bin" - Move-Item -Path WinSW.exe -Destination "$env:GITHUB_WORKSPACE\bin" - $env:PATH += ";$env:GITHUB_WORKSPACE\bin" - - cargo test --package sn-node-manager --release ` - --test upgrades ${{ matrix.test }} -- --nocapture - - # Each daemon test needs its own VM, otherwise they will interfere with each other. - node-manager-daemon-tests: - name: node manager daemon tests - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - - { - os: ubuntu-latest, - elevated: sudo env PATH="$PATH", - test: restart_node, - } - # todo: enable once url/version has been implemented for Daemon subcmd. - # - { - # os: macos-latest, - # elevated: sudo, - # test: restart_node, - # } - # - { - # os: windows-latest, - # test: restart_node, - # } - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - - name: run integration test - shell: bash - if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' - run: | - ${{ matrix.elevated }} rustup default stable - ${{ matrix.elevated }} cargo test --package sn-node-manager --release \ - --test daemon ${{ matrix.test }} -- --nocapture - - # Powershell step runs as admin by default. - - name: run integration test in powershell - if: matrix.os == 'windows-latest' - shell: pwsh - run: | - curl -L -o WinSW.exe $env:WINSW_URL - - New-Item -ItemType Directory -Force -Path "$env:GITHUB_WORKSPACE\bin" - Move-Item -Path WinSW.exe -Destination "$env:GITHUB_WORKSPACE\bin" - $env:PATH += ";$env:GITHUB_WORKSPACE\bin" - - cargo test --package sn-node-manager --release ` - --test daemon ${{ matrix.test }} -- --nocapture + # node-manager-upgrade-tests: + # name: node manager upgrade tests + # runs-on: ${{ matrix.os }} + # strategy: + # fail-fast: false + # matrix: + # include: + # - { + # os: ubuntu-latest, + # elevated: sudo env PATH="$PATH", + # test: upgrade_to_latest_version, + # } + # - { + # os: ubuntu-latest, + # elevated: sudo env PATH="$PATH", + # test: force_upgrade_when_two_binaries_have_the_same_version, + # } + # - { + # os: ubuntu-latest, + # elevated: sudo env PATH="$PATH", + # test: force_downgrade_to_a_previous_version, + # } + # - { + # os: ubuntu-latest, + # elevated: sudo env PATH="$PATH", + # test: upgrade_from_older_version_to_specific_version, + # } + # - { + # os: macos-latest, + # elevated: sudo, + # test: upgrade_to_latest_version, + # } + # - { + # os: macos-latest, + # elevated: sudo, + # test: force_upgrade_when_two_binaries_have_the_same_version, + # } + # - { + # os: macos-latest, + # elevated: sudo, + # test: force_downgrade_to_a_previous_version, + # } + # - { + # os: macos-latest, + # elevated: sudo, + # test: upgrade_from_older_version_to_specific_version, + # } + # - { os: windows-latest, test: upgrade_to_latest_version } + # - { + # os: windows-latest, + # test: force_upgrade_when_two_binaries_have_the_same_version, + # } + # - { os: windows-latest, test: force_downgrade_to_a_previous_version } + # - { + # os: windows-latest, + # test: upgrade_from_older_version_to_specific_version, + # } + # steps: + # - uses: actions/checkout@v4 + # + # - name: Install Rust + # uses: dtolnay/rust-toolchain@stable + # - uses: Swatinem/rust-cache@v2 + # + # - shell: bash + # if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' + # run: | + # ${{ matrix.elevated }} rustup default stable + # ${{ matrix.elevated }} cargo test --package sn-node-manager --release \ + # --test upgrades ${{ matrix.test }} -- --nocapture + # + # # Powershell step runs as admin by default. + # - name: run integration test in powershell + # if: matrix.os == 'windows-latest' + # shell: pwsh + # run: | + # curl -L -o WinSW.exe $env:WINSW_URL + # + # New-Item -ItemType Directory -Force -Path "$env:GITHUB_WORKSPACE\bin" + # Move-Item -Path WinSW.exe -Destination "$env:GITHUB_WORKSPACE\bin" + # $env:PATH += ";$env:GITHUB_WORKSPACE\bin" + # + # cargo test --package sn-node-manager --release ` + # --test upgrades ${{ matrix.test }} -- --nocapture + # + # # Each daemon test needs its own VM, otherwise they will interfere with each other. + # node-manager-daemon-tests: + # name: node manager daemon tests + # runs-on: ${{ matrix.os }} + # strategy: + # fail-fast: false + # matrix: + # include: + # - { + # os: ubuntu-latest, + # elevated: sudo env PATH="$PATH", + # test: restart_node, + # } + # # todo: enable once url/version has been implemented for Daemon subcmd. + # # - { + # # os: macos-latest, + # # elevated: sudo, + # # test: restart_node, + # # } + # # - { + # # os: windows-latest, + # # test: restart_node, + # # } + # steps: + # - uses: actions/checkout@v4 + # + # - name: Install Rust + # uses: dtolnay/rust-toolchain@stable + # - uses: Swatinem/rust-cache@v2 + # + # - name: run integration test + # shell: bash + # if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' + # run: | + # ${{ matrix.elevated }} rustup default stable + # ${{ matrix.elevated }} cargo test --package sn-node-manager --release \ + # --test daemon ${{ matrix.test }} -- --nocapture + # + # # Powershell step runs as admin by default. + # - name: run integration test in powershell + # if: matrix.os == 'windows-latest' + # shell: pwsh + # run: | + # curl -L -o WinSW.exe $env:WINSW_URL + # + # New-Item -ItemType Directory -Force -Path "$env:GITHUB_WORKSPACE\bin" + # Move-Item -Path WinSW.exe -Destination "$env:GITHUB_WORKSPACE\bin" + # $env:PATH += ";$env:GITHUB_WORKSPACE\bin" + # + # cargo test --package sn-node-manager --release ` + # --test daemon ${{ matrix.test }} -- --nocapture e2e: if: "!startsWith(github.event.head_commit.message, 'chore(release):')" From ff83510da343c9763bdba5132fcdbc822e4b704e Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 1 May 2024 18:12:42 +0100 Subject: [PATCH 198/205] ci: add `node-launchpad` binary to release process The new TUI for the node manager is a new, separate binary that is being added in to the release process. It will have a Github release. For some reason it seems to have became necessary to explicitly call `rustup target` during the artifacts build process. Without doing it, I was getting build errors on macOS. --- .github/workflows/build-release-artifacts.yml | 8 ++++---- .github/workflows/release.yml | 10 +++++++--- Justfile | 19 +++++++++++++++++-- release-plz.toml | 12 ++++++++++++ 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-release-artifacts.yml b/.github/workflows/build-release-artifacts.yml index f2b5452300..1d67e74c90 100644 --- a/.github/workflows/build-release-artifacts.yml +++ b/.github/workflows/build-release-artifacts.yml @@ -10,7 +10,7 @@ on: default: main env: - JUST_BIN_URL: https://github.com/casey/just/releases/download/1.13.0/just-1.13.0-x86_64-unknown-linux-musl.tar.gz + JUST_BIN_URL: https://github.com/casey/just/releases/download/1.25.2/just-1.25.2-x86_64-unknown-linux-musl.tar.gz jobs: build: @@ -95,8 +95,8 @@ jobs: run: | curl -L -O $JUST_BIN_URL mkdir just - tar xvf just-1.13.0-x86_64-unknown-linux-musl.tar.gz -C just - rm just-1.13.0-x86_64-unknown-linux-musl.tar.gz + tar xvf just-1.25.2-x86_64-unknown-linux-musl.tar.gz -C just + rm just-1.25.2-x86_64-unknown-linux-musl.tar.gz sudo mv just/just /usr/local/bin rm -rf just sudo apt-get install -y tree @@ -106,9 +106,9 @@ jobs: tree artifacts just package-release-assets "safe" just package-release-assets "safenode" - just package-release-assets "testnet" just package-release-assets "faucet" just package-release-assets "safenode_rpc_client" + just package-release-assets "node-launchpad" - uses: actions/upload-artifact@main with: name: packaged_binaries diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c593365580..5bd7eef54a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ on: env: RELEASE_PLZ_BIN_URL: https://github.com/MarcoIeni/release-plz/releases/download/release-plz-v0.3.43/release-plz-x86_64-unknown-linux-gnu.tar.gz - JUST_BIN_URL: https://github.com/casey/just/releases/download/1.13.0/just-1.13.0-x86_64-unknown-linux-musl.tar.gz + JUST_BIN_URL: https://github.com/casey/just/releases/download/1.25.2/just-1.25.2-x86_64-unknown-linux-musl.tar.gz WORKFLOW_URL: https://github.com/maidsafe/safe_network/actions/runs jobs: @@ -133,8 +133,8 @@ jobs: curl -L -O $JUST_BIN_URL mkdir just - tar xvf just-1.13.0-x86_64-unknown-linux-musl.tar.gz -C just - rm just-1.13.0-x86_64-unknown-linux-musl.tar.gz + tar xvf just-1.25.2-x86_64-unknown-linux-musl.tar.gz -C just + rm just-1.25.2-x86_64-unknown-linux-musl.tar.gz sudo mv just/just /usr/local/bin rm -rf just @@ -158,12 +158,14 @@ jobs: just package-release-assets "safenode_rpc_client" just package-release-assets "safenode-manager" just package-release-assets "safenodemand" + just package-release-assets "node-launcher" just upload-release-assets-to-s3 "safe" just upload-release-assets-to-s3 "safenode" just upload-release-assets-to-s3 "safenode-manager" just upload-release-assets-to-s3 "safenodemand" just upload-release-assets-to-s3 "faucet" just upload-release-assets-to-s3 "safenode_rpc_client" + just upload-release-assets-to-s3 "node-launcher" # The `release-plz` command publishes crates which had their versions bumped, and also # creates Github releases. The binaries are then attached to the releases in the @@ -184,12 +186,14 @@ jobs: just package-release-assets "safenode_rpc_client" "latest" just package-release-assets "safenode-manager" "latest" just package-release-assets "safenodemand" "latest" + just package-release-assets "node-launcher" "latest" just upload-release-assets-to-s3 "safe" just upload-release-assets-to-s3 "safenode" just upload-release-assets-to-s3 "safenode-manager" just upload-release-assets-to-s3 "safenodemand" just upload-release-assets-to-s3 "faucet" just upload-release-assets-to-s3 "safenode_rpc_client" + just upload-release-assets-to-s3 "node-launcher" - name: post notification to slack on failure if: ${{ failure() }} diff --git a/Justfile b/Justfile index 2b32a3a2ae..a3273df8ca 100644 --- a/Justfile +++ b/Justfile @@ -98,9 +98,10 @@ build-release-artifacts arch: sudo apt update -y sudo apt-get install -y musl-tools fi - rustup target add x86_64-unknown-linux-musl fi + rustup target add {{arch}} + rm -rf artifacts mkdir artifacts cargo clean @@ -118,6 +119,7 @@ build-release-artifacts arch: cross build --release --target $arch --bin safenodemand cross build --release --target $arch --bin faucet --features=distribution cross build --release --target $arch --bin safenode_rpc_client + cross build --release --target $arch --bin node-launchpad else cargo build --release --features="network-contacts,distribution" --target $arch --bin safe cargo build --release --features=network-contacts --target $arch --bin safenode @@ -125,6 +127,7 @@ build-release-artifacts arch: cargo build --release --target $arch --bin safenodemand cargo build --release --target $arch --bin faucet --features=distribution cargo build --release --target $arch --bin safenode_rpc_client + cargo build --release --target $arch --bin node-launchpad fi find target/$arch/release -maxdepth 1 -type f -exec cp '{}' artifacts \; @@ -168,7 +171,8 @@ package-release-assets bin version="": bin="{{bin}}" - supported_bins=("safe" "safenode" "safenode-manager" "safenodemand" "faucet" "safenode_rpc_client") + supported_bins=(\ + "safe" "safenode" "safenode-manager" "safenodemand" "faucet" "safenode_rpc_client" "node-launchpad") crate_dir_name="" # In the case of the node manager, the actual name of the crate is `sn-node-manager`, but the @@ -193,6 +197,9 @@ package-release-assets bin version="": safenode_rpc_client) crate_dir_name="sn_node_rpc_client" ;; + node-launchpad) + crate_dir_name="sn_node_launchpad" + ;; *) echo "The $bin binary is not supported" exit 1 @@ -234,6 +241,7 @@ upload-github-release-assets: "sn-node-manager" "sn_faucet" "sn_node_rpc_client" + "sn_node_launchpad" ) commit_msg=$(git log -1 --pretty=%B) @@ -270,6 +278,10 @@ upload-github-release-assets: bin_name="safenode_rpc_client" bucket="sn-node-rpc-client" ;; + sn_node_launchpad) + bin_name="node-launchpad" + bucket="sn-node-launchpad" + ;; *) echo "The $crate crate is not supported" exit 1 @@ -316,6 +328,9 @@ upload-release-assets-to-s3 bin_name: safenode_rpc_client) bucket="sn-node-rpc-client" ;; + node-launchpad) + bucket="sn-node-launchpad" + ;; *) echo "The {{bin_name}} binary is not supported" exit 1 diff --git a/release-plz.toml b/release-plz.toml index afcbba7a71..e2c3711fa0 100644 --- a/release-plz.toml +++ b/release-plz.toml @@ -7,6 +7,12 @@ publish_allow_dirty = false semver_check = false git_release_type = "auto" +[[package]] +name = "sn_auditor" +changelog_update = true +git_release_enable = false +publish = true + [[package]] name = "sn_build_info" changelog_update = true @@ -75,6 +81,12 @@ changelog_include = [ "sn_protocol", ] +[[package]] +name = "sn_node_launchpad" +changelog_update = true +git_release_enable = true +publish = true + [[package]] name = "sn_node_rpc_client" changelog_update = true From ebac38224d922a57d102e57963764b7d37fc1812 Mon Sep 17 00:00:00 2001 From: qima Date: Tue, 30 Apr 2024 18:14:27 +0800 Subject: [PATCH 199/205] feat(cli): track spend creation reasons during audit --- .github/workflows/memcheck.yml | 17 +++++ sn_client/src/audit/spend_dag.rs | 81 +++++++++++++++++++--- sn_transfers/src/cashnotes/signed_spend.rs | 6 ++ 3 files changed, 93 insertions(+), 11 deletions(-) diff --git a/.github/workflows/memcheck.yml b/.github/workflows/memcheck.yml index d7e089b4da..5c8f7f5e57 100644 --- a/.github/workflows/memcheck.yml +++ b/.github/workflows/memcheck.yml @@ -279,6 +279,15 @@ jobs: SN_LOG: "all" timeout-minutes: 10 + - name: Audit from genesis to collect entire spend DAG and dump to a dot file + run: | + cargo run --bin safe --release -- --log-output-dest=data-dir wallet audit --dot > spend_dag.dot + echo "==============================================================================" + cat spend_dag.dot + env: + SN_LOG: "all" + timeout-minutes: 60 + - name: Check nodes running shell: bash timeout-minutes: 1 @@ -436,3 +445,11 @@ jobs: path: faucet_log.log continue-on-error: true if: always() + + - name: Upload spend DAG + uses: actions/upload-artifact@main + with: + name: memory_check_spend_dag + path: spend_dag.dot + continue-on-error: true + if: always() diff --git a/sn_client/src/audit/spend_dag.rs b/sn_client/src/audit/spend_dag.rs index 0b5053273f..6d2c99106f 100644 --- a/sn_client/src/audit/spend_dag.rs +++ b/sn_client/src/audit/spend_dag.rs @@ -11,11 +11,40 @@ use petgraph::graph::{DiGraph, NodeIndex}; use petgraph::visit::EdgeRef; use serde::{Deserialize, Serialize}; use sn_transfers::{is_genesis_spend, CashNoteRedemption, NanoTokens, SignedSpend, SpendAddress}; -use std::collections::{BTreeMap, BTreeSet}; -use std::path::Path; +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt, + path::Path, +}; use super::dag_error::{DagError, SpendFault}; +#[derive(Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub struct SpendDagAddress { + address: SpendAddress, + purpose: String, +} + +impl Clone for SpendDagAddress { + fn clone(&self) -> Self { + SpendDagAddress { + address: self.address, + purpose: self.purpose.clone(), + } + } +} + +impl std::fmt::Debug for SpendDagAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "address: {}\npurpose: {}", + &self.address.to_hex()[0..6], + self.purpose + ) + } +} + /// A DAG representing the spends from a specific Spend all the way to the UTXOs. /// Starting from Genesis, this would encompass all the spends that have happened on the network /// at a certain point in time. @@ -33,13 +62,15 @@ use super::dag_error::{DagError, SpendFault}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SpendDag { /// A directed graph of spend addresses - dag: DiGraph, + dag: DiGraph, /// All the spends refered to in the dag indexed by their SpendAddress spends: BTreeMap, /// The source of the DAG (aka Genesis) source: SpendAddress, /// Recorded faults in the DAG faults: BTreeMap>, + /// Creation reasons (purpose) of the spends + creation_reasons: BTreeMap, } type DagIndex = usize; @@ -91,6 +122,7 @@ impl SpendDag { spends: BTreeMap::new(), source, faults: BTreeMap::new(), + creation_reasons: BTreeMap::new(), } } @@ -110,17 +142,42 @@ impl SpendDag { Ok(()) } + fn get_dag_address(&self, spend_addr: &SpendAddress) -> SpendDagAddress { + let spend_creation_reason = + if let Some(creation_reason) = self.creation_reasons.get(spend_addr) { + creation_reason.clone() + } else { + "Undefined".to_string() + }; + SpendDagAddress { + address: *spend_addr, + purpose: spend_creation_reason, + } + } + /// Insert a spend into the dag /// Creating edges (links) from its ancestors and to its descendants /// If the inserted spend is already known, it will be ignored /// If the inserted spend is a double spend, it will be saved along with the previous spend /// Return true if the spend was inserted and false if it was already in the DAG pub fn insert(&mut self, spend_addr: SpendAddress, spend: SignedSpend) -> bool { + for output in spend.outputs().iter() { + let output_spend_address = SpendAddress::from_unique_pubkey(&output.unique_pubkey); + if let Some(old) = self + .creation_reasons + .insert(output_spend_address, output.purpose.clone()) + { + error!("Spend {output_spend_address:?} already have an un-expected creation reason: {old:?}"); + } + } + + let spend_dag_address = self.get_dag_address(&spend_addr); + let existing_entry = self.spends.get(&spend_addr).cloned(); let new_node_idx = match existing_entry { // add new spend to the DAG None => { - let node_idx = self.dag.add_node(spend_addr); + let node_idx = self.dag.add_node(spend_dag_address); self.spends.insert( spend_addr, DagEntry::Spend(Box::new(spend.clone()), node_idx.index()), @@ -142,7 +199,7 @@ impl SpendDag { return false; } - let node_idx = self.dag.add_node(spend_addr); + let node_idx = self.dag.add_node(spend_dag_address); let double_spend = DagEntry::DoubleSpend(vec![ (existing_spend.clone(), idx), (spend.clone(), node_idx.index()), @@ -156,7 +213,7 @@ impl SpendDag { return false; } - let node_idx = self.dag.add_node(spend_addr); + let node_idx = self.dag.add_node(spend_dag_address); let mut vec_s = vec_s.clone(); vec_s.push((spend.clone(), node_idx.index())); self.spends.insert(spend_addr, DagEntry::DoubleSpend(vec_s)); @@ -167,10 +224,11 @@ impl SpendDag { // link to descendants for descendant in spend.spend.spent_tx.outputs.iter() { let descendant_addr = SpendAddress::from_unique_pubkey(&descendant.unique_pubkey); + let descendant_dag_address = self.get_dag_address(&descendant_addr); // add descendant if not already in dag let spends_at_addr = self.spends.entry(descendant_addr).or_insert_with(|| { - let node_idx = self.dag.add_node(descendant_addr); + let node_idx = self.dag.add_node(descendant_dag_address); DagEntry::NotGatheredYet(node_idx.index()) }); @@ -191,10 +249,11 @@ impl SpendDag { const PENDING_AMOUNT: NanoTokens = NanoTokens::from(0); for ancestor in spend.spend.parent_tx.inputs.iter() { let ancestor_addr = SpendAddress::from_unique_pubkey(&ancestor.unique_pubkey); + let ancestor_dag_address = self.get_dag_address(&ancestor_addr); // add ancestor if not already in dag let spends_at_addr = self.spends.entry(ancestor_addr).or_insert_with(|| { - let node_idx = self.dag.add_node(ancestor_addr); + let node_idx = self.dag.add_node(ancestor_dag_address); DagEntry::NotGatheredYet(node_idx.index()) }); @@ -265,7 +324,7 @@ impl SpendDag { .neighbors_directed(node_index, petgraph::Direction::Outgoing) .any(|_| true) { - let utxo_addr = self.dag[node_index]; + let utxo_addr = self.dag[node_index].address; leaves.insert(utxo_addr); } } @@ -456,7 +515,7 @@ impl SpendDag { .flat_map(|idx| { self.dag .neighbors_directed(NodeIndex::new(idx), petgraph::Direction::Outgoing) - .map(|i| &self.dag[i]) + .map(|i| &self.dag[i].address) }) .collect(); @@ -494,7 +553,7 @@ impl SpendDag { self.dag .neighbors_directed(NodeIndex::new(idx), petgraph::Direction::Incoming) }) - .map(|parent_idx| &self.dag[parent_idx]) + .map(|parent_idx| &self.dag[parent_idx].address) .collect(); let non_orphans = BTreeSet::from_iter(all_descendants.into_iter().chain(parents).chain([source])); diff --git a/sn_transfers/src/cashnotes/signed_spend.rs b/sn_transfers/src/cashnotes/signed_spend.rs index d0b71f4a5b..2d0ebb6cdf 100644 --- a/sn_transfers/src/cashnotes/signed_spend.rs +++ b/sn_transfers/src/cashnotes/signed_spend.rs @@ -7,6 +7,7 @@ // permissions and limitations relating to use of the SAFE Network Software. use super::{Hash, NanoTokens, Transaction, UniquePubkey}; +use crate::cashnotes::transaction::Output; use crate::{DerivationIndex, Result, Signature, SpendAddress, TransferError}; use custom_debug::Debug; @@ -183,6 +184,11 @@ impl SignedSpend { trace!("Validated parent_spends for {unique_key}"); Ok(()) } + + /// Get a reference to the outputs + pub fn outputs(&self) -> &Vec { + &self.spend.spent_tx.outputs + } } // Impl manually to avoid clippy complaint about Hash conflict. From 4969beab836760c6081bd50718789cff422cfec1 Mon Sep 17 00:00:00 2001 From: qima Date: Tue, 30 Apr 2024 23:08:22 +0800 Subject: [PATCH 200/205] chore: refactor CASH_NOTE_REASON strings to consts --- Cargo.lock | 1 + sn_cli/Cargo.toml | 1 + sn_cli/src/bin/subcommands/wallet/wo_wallet.rs | 3 ++- sn_client/src/audit/spend_dag.rs | 9 ++++++--- sn_client/src/wallet.rs | 3 ++- sn_transfers/src/cashnotes.rs | 5 +++++ sn_transfers/src/genesis.rs | 3 ++- sn_transfers/src/lib.rs | 3 ++- sn_transfers/src/transfers/offline_transfer.rs | 6 +++--- sn_transfers/src/wallet/hot_wallet.rs | 4 ++-- 10 files changed, 26 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0ba158c686..03d627fd86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6876,6 +6876,7 @@ dependencies = [ "sn_logging", "sn_peers_acquisition", "sn_protocol", + "sn_transfers", "tempfile", "tiny-keccak", "tokio", diff --git a/sn_cli/Cargo.toml b/sn_cli/Cargo.toml index f4f2c3990d..08dd6f3051 100644 --- a/sn_cli/Cargo.toml +++ b/sn_cli/Cargo.toml @@ -62,6 +62,7 @@ sn_client = { path = "../sn_client", version = "0.105.3" } sn_logging = { path = "../sn_logging", version = "0.2.25" } sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.2.10" } sn_protocol = { path = "../sn_protocol", version = "0.16.3" } +sn_transfers = { path = "../sn_transfers", version = "0.17.2" } tempfile = "3.6.0" tiny-keccak = "~2.0.2" tokio = { version = "1.32.0", features = [ diff --git a/sn_cli/src/bin/subcommands/wallet/wo_wallet.rs b/sn_cli/src/bin/subcommands/wallet/wo_wallet.rs index 99e694d7b2..0503ab56d3 100644 --- a/sn_cli/src/bin/subcommands/wallet/wo_wallet.rs +++ b/sn_cli/src/bin/subcommands/wallet/wo_wallet.rs @@ -20,6 +20,7 @@ use sn_client::transfers::{ WatchOnlyWallet, }; use sn_client::Client; +use sn_transfers::CASH_NOTE_PURPOSE_FOR_TRANSFER; use std::{ collections::{BTreeMap, BTreeSet}, path::Path, @@ -233,7 +234,7 @@ fn build_unsigned_transaction(from: &str, amount: &str, to: &str, root_dir: &Pat }; let unsigned_transfer = wallet.build_unsigned_transaction( - vec![("CASH_NOTE_REASON_FOR_TRANSFER".to_string(), amount, to)], + vec![(CASH_NOTE_PURPOSE_FOR_TRANSFER.to_string(), amount, to)], None, )?; diff --git a/sn_client/src/audit/spend_dag.rs b/sn_client/src/audit/spend_dag.rs index 6d2c99106f..33a362726f 100644 --- a/sn_client/src/audit/spend_dag.rs +++ b/sn_client/src/audit/spend_dag.rs @@ -10,7 +10,10 @@ use petgraph::dot::Dot; use petgraph::graph::{DiGraph, NodeIndex}; use petgraph::visit::EdgeRef; use serde::{Deserialize, Serialize}; -use sn_transfers::{is_genesis_spend, CashNoteRedemption, NanoTokens, SignedSpend, SpendAddress}; +use sn_transfers::{ + is_genesis_spend, CashNoteRedemption, NanoTokens, SignedSpend, SpendAddress, + CASH_NOTE_PURPOSE_FOR_GENESIS, CASH_NOTE_PURPOSE_FOR_NETWORK_ROYALTIES, +}; use std::{ collections::{BTreeMap, BTreeSet}, fmt, @@ -147,7 +150,7 @@ impl SpendDag { if let Some(creation_reason) = self.creation_reasons.get(spend_addr) { creation_reason.clone() } else { - "Undefined".to_string() + CASH_NOTE_PURPOSE_FOR_GENESIS.to_string() }; SpendDagAddress { address: *spend_addr, @@ -409,7 +412,7 @@ impl SpendDag { royalties.push(CashNoteRedemption::new( *derivation_idx, spend_addr, - "CASH_NOTE_REASON_FOR_NETWORK_ROYALTIES".to_string(), + CASH_NOTE_PURPOSE_FOR_NETWORK_ROYALTIES.to_string(), )); } } diff --git a/sn_client/src/wallet.rs b/sn_client/src/wallet.rs index 1f60f09d1e..07c77f2247 100644 --- a/sn_client/src/wallet.rs +++ b/sn_client/src/wallet.rs @@ -18,6 +18,7 @@ use sn_protocol::NetworkAddress; use sn_transfers::{ CashNote, CashNoteOutputDetails, HotWallet, MainPubkey, NanoTokens, Payment, PaymentQuote, SignedSpend, SpendAddress, Transaction, Transfer, UniquePubkey, WalletError, WalletResult, + CASH_NOTE_PURPOSE_FOR_TRANSFER, }; use std::{ collections::{BTreeMap, BTreeSet}, @@ -307,7 +308,7 @@ impl WalletClient { verify_store: bool, ) -> WalletResult { let created_cash_notes = self.wallet.local_send( - vec![("CASH_NOTE_REASON_FOR_TRANSFER".to_string(), amount, to)], + vec![(CASH_NOTE_PURPOSE_FOR_TRANSFER.to_string(), amount, to)], None, )?; diff --git a/sn_transfers/src/cashnotes.rs b/sn_transfers/src/cashnotes.rs index baacd00590..a1caa843ea 100644 --- a/sn_transfers/src/cashnotes.rs +++ b/sn_transfers/src/cashnotes.rs @@ -15,6 +15,11 @@ mod signed_spend; mod transaction; mod unique_keys; +pub const CASH_NOTE_PURPOSE_FOR_GENESIS: &str = "GENESIS"; +pub const CASH_NOTE_PURPOSE_FOR_NETWORK_ROYALTIES: &str = "ROYALTY"; +pub const CASH_NOTE_PURPOSE_FOR_CHANGE: &str = "CHANGE"; +pub const CASH_NOTE_PURPOSE_FOR_TRANSFER: &str = "TRANSFER"; + pub(crate) use builder::{CashNoteBuilder, TransactionBuilder}; pub(crate) use transaction::Input; diff --git a/sn_transfers/src/genesis.rs b/sn_transfers/src/genesis.rs index 2b02b68049..c40e8b8435 100644 --- a/sn_transfers/src/genesis.rs +++ b/sn_transfers/src/genesis.rs @@ -8,6 +8,7 @@ use super::wallet::HotWallet; +use crate::cashnotes::CASH_NOTE_PURPOSE_FOR_GENESIS; use crate::{ CashNote, DerivationIndex, Hash, Input, MainPubkey, MainSecretKey, NanoTokens, SignedSpend, Transaction, TransactionBuilder, TransferError as CashNoteError, @@ -159,7 +160,7 @@ pub fn create_first_cash_note_from_key( ) .add_output( NanoTokens::from(GENESIS_CASHNOTE_AMOUNT), - "CASH_NOTE_REASON_FOR_GENESIS".to_string(), + CASH_NOTE_PURPOSE_FOR_GENESIS.to_string(), main_pubkey, derivation_index, ) diff --git a/sn_transfers/src/lib.rs b/sn_transfers/src/lib.rs index 90dc0cf21e..474820bf30 100644 --- a/sn_transfers/src/lib.rs +++ b/sn_transfers/src/lib.rs @@ -21,7 +21,8 @@ pub(crate) use cashnotes::{Input, TransactionBuilder}; pub use cashnotes::{ CashNote, CashNoteOutputDetails, DerivationIndex, DerivedSecretKey, Hash, MainPubkey, MainSecretKey, NanoTokens, SignedSpend, Spend, SpendAddress, Transaction, UniquePubkey, - UnsignedTransfer, + UnsignedTransfer, CASH_NOTE_PURPOSE_FOR_GENESIS, CASH_NOTE_PURPOSE_FOR_NETWORK_ROYALTIES, + CASH_NOTE_PURPOSE_FOR_TRANSFER, }; pub use error::{Result, TransferError}; pub use transfers::{CashNoteRedemption, OfflineTransfer, Transfer}; diff --git a/sn_transfers/src/transfers/offline_transfer.rs b/sn_transfers/src/transfers/offline_transfer.rs index a81d9130ab..6551768fa4 100644 --- a/sn_transfers/src/transfers/offline_transfer.rs +++ b/sn_transfers/src/transfers/offline_transfer.rs @@ -7,7 +7,7 @@ // permissions and limitations relating to use of the SAFE Network Software. use crate::{ - cashnotes::{CashNoteBuilder, UnsignedTransfer}, + cashnotes::{CashNoteBuilder, UnsignedTransfer, CASH_NOTE_PURPOSE_FOR_CHANGE}, rng, CashNote, CashNoteOutputDetails, DerivationIndex, DerivedSecretKey, Hash, Input, MainPubkey, NanoTokens, Result, SignedSpend, Transaction, TransactionBuilder, TransferError, UniquePubkey, NETWORK_ROYALTIES_PK, @@ -19,7 +19,7 @@ use std::collections::{BTreeMap, BTreeSet}; /// List of CashNotes, with (optionally when needed) their corresponding derived owning secret key. pub type CashNotesAndSecretKey = Vec<(CashNote, Option)>; -/// RecipientDetails: (amount, cash_note_reason, pub_key, derivation_index) +/// RecipientDetails: (amount, cash_note_purpose, pub_key, derivation_index) pub type TransferRecipientDetails = (NanoTokens, String, MainPubkey, DerivationIndex); /// Offline Transfer @@ -277,7 +277,7 @@ fn create_transaction_builder_with( if !change.is_zero() { tx_builder = tx_builder.add_output( change, - "CASH_NOTE_REASON_FOR_CHANGE".to_string(), + CASH_NOTE_PURPOSE_FOR_CHANGE.to_string(), change_to, derivation_index, ); diff --git a/sn_transfers/src/wallet/hot_wallet.rs b/sn_transfers/src/wallet/hot_wallet.rs index 55a0ff4999..9be2454e6a 100644 --- a/sn_transfers/src/wallet/hot_wallet.rs +++ b/sn_transfers/src/wallet/hot_wallet.rs @@ -24,7 +24,7 @@ use crate::{ transfers::{CashNotesAndSecretKey, OfflineTransfer}, CashNote, CashNoteOutputDetails, CashNoteRedemption, DerivationIndex, DerivedSecretKey, Hash, MainPubkey, MainSecretKey, NanoTokens, SignedSpend, Spend, Transaction, Transfer, UniquePubkey, - WalletError, NETWORK_ROYALTIES_PK, + WalletError, CASH_NOTE_PURPOSE_FOR_NETWORK_ROYALTIES, NETWORK_ROYALTIES_PK, }; use std::{ collections::{BTreeMap, BTreeSet, HashSet}, @@ -402,7 +402,7 @@ impl HotWallet { let royalties_fee = calculate_royalties_fee(quote.cost); let royalties_payee = ( royalties_fee, - "CASH_NOTE_REASON_FOR_NETWORK_ROYALTIES".to_string(), + CASH_NOTE_PURPOSE_FOR_NETWORK_ROYALTIES.to_string(), *NETWORK_ROYALTIES_PK, DerivationIndex::random(&mut rng), ); From cc04419f4e6ada8aee291f7bd77096c54b31a876 Mon Sep 17 00:00:00 2001 From: qima Date: Wed, 1 May 2024 17:39:01 +0800 Subject: [PATCH 201/205] feat(client): dump spends creation_reason statistics --- .github/workflows/memcheck.yml | 10 ++++---- sn_cli/src/bin/subcommands/wallet/audit.rs | 7 ++++++ sn_client/src/audit/spend_dag.rs | 29 +++++++++++++++++----- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/.github/workflows/memcheck.yml b/.github/workflows/memcheck.yml index 5c8f7f5e57..b895dae3e7 100644 --- a/.github/workflows/memcheck.yml +++ b/.github/workflows/memcheck.yml @@ -281,9 +281,9 @@ jobs: - name: Audit from genesis to collect entire spend DAG and dump to a dot file run: | - cargo run --bin safe --release -- --log-output-dest=data-dir wallet audit --dot > spend_dag.dot + cargo run --bin safe --release -- --log-output-dest=data-dir wallet audit --dot > spend_dag_and_statistics.txt echo "==============================================================================" - cat spend_dag.dot + cat spend_dag_and_statistics.txt env: SN_LOG: "all" timeout-minutes: 60 @@ -446,10 +446,10 @@ jobs: continue-on-error: true if: always() - - name: Upload spend DAG + - name: Upload spend DAG and statistics uses: actions/upload-artifact@main with: - name: memory_check_spend_dag - path: spend_dag.dot + name: memory_check_spend_dag_and_statistics + path: spend_dag_and_statistics.txt continue-on-error: true if: always() diff --git a/sn_cli/src/bin/subcommands/wallet/audit.rs b/sn_cli/src/bin/subcommands/wallet/audit.rs index ca6c8adb15..b05d09bc90 100644 --- a/sn_cli/src/bin/subcommands/wallet/audit.rs +++ b/sn_cli/src/bin/subcommands/wallet/audit.rs @@ -40,7 +40,14 @@ async fn gather_spend_dag(client: &Client, root_dir: &Path) -> Result pub async fn audit(client: &Client, to_dot: bool, royalties: bool, root_dir: &Path) -> Result<()> { if to_dot { let dag = gather_spend_dag(client, root_dir).await?; + println!( + "========================== spends DAG diagraph =============================" + ); println!("{}", dag.dump_dot_format()); + println!( + "======================= spends purpose statistics ==========================" + ); + println!("{}", dag.dump_creation_reasons_statistics()); } else if royalties { let dag = gather_spend_dag(client, root_dir).await?; let royalties = dag.all_royalties()?; diff --git a/sn_client/src/audit/spend_dag.rs b/sn_client/src/audit/spend_dag.rs index 33a362726f..fe8935a745 100644 --- a/sn_client/src/audit/spend_dag.rs +++ b/sn_client/src/audit/spend_dag.rs @@ -73,7 +73,7 @@ pub struct SpendDag { /// Recorded faults in the DAG faults: BTreeMap>, /// Creation reasons (purpose) of the spends - creation_reasons: BTreeMap, + creation_reasons: BTreeMap, } type DagIndex = usize; @@ -147,7 +147,7 @@ impl SpendDag { fn get_dag_address(&self, spend_addr: &SpendAddress) -> SpendDagAddress { let spend_creation_reason = - if let Some(creation_reason) = self.creation_reasons.get(spend_addr) { + if let Some((creation_reason, _amount)) = self.creation_reasons.get(spend_addr) { creation_reason.clone() } else { CASH_NOTE_PURPOSE_FOR_GENESIS.to_string() @@ -166,10 +166,10 @@ impl SpendDag { pub fn insert(&mut self, spend_addr: SpendAddress, spend: SignedSpend) -> bool { for output in spend.outputs().iter() { let output_spend_address = SpendAddress::from_unique_pubkey(&output.unique_pubkey); - if let Some(old) = self - .creation_reasons - .insert(output_spend_address, output.purpose.clone()) - { + if let Some(old) = self.creation_reasons.insert( + output_spend_address, + (output.purpose.clone(), output.amount), + ) { error!("Spend {output_spend_address:?} already have an un-expected creation reason: {old:?}"); } } @@ -338,6 +338,23 @@ impl SpendDag { format!("{:?}", Dot::with_config(&self.dag, &[])) } + pub fn dump_creation_reasons_statistics(&self) -> String { + let mut statistics: BTreeMap> = Default::default(); + for (_spend_addr, (reason, amount)) in self.creation_reasons.iter() { + let holders = statistics.entry(reason.clone()).or_default(); + holders.push(*amount); + } + let mut content = "Purpose,Times,Amount".to_string(); + for (purpose, payments) in statistics.iter() { + let total_amount: u64 = payments + .iter() + .map(|nano_tokens| nano_tokens.as_nano()) + .sum(); + content = format!("{content}\n{purpose},{},{total_amount}", payments.len()); + } + content + } + /// Merges the given dag into ours pub fn merge(&mut self, sub_dag: SpendDag) -> Result<(), DagError> { let source = self.source(); From 8d7d9c5062056c2ce52edeeb3cabe002317517d9 Mon Sep 17 00:00:00 2001 From: qima Date: Thu, 2 May 2024 00:06:40 +0800 Subject: [PATCH 202/205] chore: address review comments --- sn_cli/src/bin/subcommands/wallet/wo_wallet.rs | 4 ++-- sn_client/src/audit/spend_dag.rs | 17 ++++------------- sn_client/src/wallet.rs | 4 ++-- sn_transfers/src/cashnotes.rs | 8 ++++---- sn_transfers/src/genesis.rs | 4 ++-- sn_transfers/src/lib.rs | 4 ++-- sn_transfers/src/transfers/offline_transfer.rs | 4 ++-- sn_transfers/src/wallet/hot_wallet.rs | 4 ++-- 8 files changed, 20 insertions(+), 29 deletions(-) diff --git a/sn_cli/src/bin/subcommands/wallet/wo_wallet.rs b/sn_cli/src/bin/subcommands/wallet/wo_wallet.rs index 0503ab56d3..3659312b6a 100644 --- a/sn_cli/src/bin/subcommands/wallet/wo_wallet.rs +++ b/sn_cli/src/bin/subcommands/wallet/wo_wallet.rs @@ -20,7 +20,7 @@ use sn_client::transfers::{ WatchOnlyWallet, }; use sn_client::Client; -use sn_transfers::CASH_NOTE_PURPOSE_FOR_TRANSFER; +use sn_transfers::CASHNOTE_PURPOSE_OF_TRANSFER; use std::{ collections::{BTreeMap, BTreeSet}, path::Path, @@ -234,7 +234,7 @@ fn build_unsigned_transaction(from: &str, amount: &str, to: &str, root_dir: &Pat }; let unsigned_transfer = wallet.build_unsigned_transaction( - vec![(CASH_NOTE_PURPOSE_FOR_TRANSFER.to_string(), amount, to)], + vec![(CASHNOTE_PURPOSE_OF_TRANSFER.to_string(), amount, to)], None, )?; diff --git a/sn_client/src/audit/spend_dag.rs b/sn_client/src/audit/spend_dag.rs index fe8935a745..1e059e77fb 100644 --- a/sn_client/src/audit/spend_dag.rs +++ b/sn_client/src/audit/spend_dag.rs @@ -12,7 +12,7 @@ use petgraph::visit::EdgeRef; use serde::{Deserialize, Serialize}; use sn_transfers::{ is_genesis_spend, CashNoteRedemption, NanoTokens, SignedSpend, SpendAddress, - CASH_NOTE_PURPOSE_FOR_GENESIS, CASH_NOTE_PURPOSE_FOR_NETWORK_ROYALTIES, + CASHNOTE_PURPOSE_OF_GENESIS, CASHNOTE_PURPOSE_OF_NETWORK_ROYALTIES, }; use std::{ collections::{BTreeMap, BTreeSet}, @@ -22,21 +22,12 @@ use std::{ use super::dag_error::{DagError, SpendFault}; -#[derive(Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] pub struct SpendDagAddress { address: SpendAddress, purpose: String, } -impl Clone for SpendDagAddress { - fn clone(&self) -> Self { - SpendDagAddress { - address: self.address, - purpose: self.purpose.clone(), - } - } -} - impl std::fmt::Debug for SpendDagAddress { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::fmt::Result { write!( @@ -150,7 +141,7 @@ impl SpendDag { if let Some((creation_reason, _amount)) = self.creation_reasons.get(spend_addr) { creation_reason.clone() } else { - CASH_NOTE_PURPOSE_FOR_GENESIS.to_string() + CASHNOTE_PURPOSE_OF_GENESIS.to_string() }; SpendDagAddress { address: *spend_addr, @@ -429,7 +420,7 @@ impl SpendDag { royalties.push(CashNoteRedemption::new( *derivation_idx, spend_addr, - CASH_NOTE_PURPOSE_FOR_NETWORK_ROYALTIES.to_string(), + CASHNOTE_PURPOSE_OF_NETWORK_ROYALTIES.to_string(), )); } } diff --git a/sn_client/src/wallet.rs b/sn_client/src/wallet.rs index 07c77f2247..ba034b7a82 100644 --- a/sn_client/src/wallet.rs +++ b/sn_client/src/wallet.rs @@ -18,7 +18,7 @@ use sn_protocol::NetworkAddress; use sn_transfers::{ CashNote, CashNoteOutputDetails, HotWallet, MainPubkey, NanoTokens, Payment, PaymentQuote, SignedSpend, SpendAddress, Transaction, Transfer, UniquePubkey, WalletError, WalletResult, - CASH_NOTE_PURPOSE_FOR_TRANSFER, + CASHNOTE_PURPOSE_OF_TRANSFER, }; use std::{ collections::{BTreeMap, BTreeSet}, @@ -308,7 +308,7 @@ impl WalletClient { verify_store: bool, ) -> WalletResult { let created_cash_notes = self.wallet.local_send( - vec![(CASH_NOTE_PURPOSE_FOR_TRANSFER.to_string(), amount, to)], + vec![(CASHNOTE_PURPOSE_OF_TRANSFER.to_string(), amount, to)], None, )?; diff --git a/sn_transfers/src/cashnotes.rs b/sn_transfers/src/cashnotes.rs index a1caa843ea..8ee0c80d2d 100644 --- a/sn_transfers/src/cashnotes.rs +++ b/sn_transfers/src/cashnotes.rs @@ -15,10 +15,10 @@ mod signed_spend; mod transaction; mod unique_keys; -pub const CASH_NOTE_PURPOSE_FOR_GENESIS: &str = "GENESIS"; -pub const CASH_NOTE_PURPOSE_FOR_NETWORK_ROYALTIES: &str = "ROYALTY"; -pub const CASH_NOTE_PURPOSE_FOR_CHANGE: &str = "CHANGE"; -pub const CASH_NOTE_PURPOSE_FOR_TRANSFER: &str = "TRANSFER"; +pub const CASHNOTE_PURPOSE_OF_GENESIS: &str = "GENESIS"; +pub const CASHNOTE_PURPOSE_OF_NETWORK_ROYALTIES: &str = "ROYALTY"; +pub const CASHNOTE_PURPOSE_OF_CHANGE: &str = "CHANGE"; +pub const CASHNOTE_PURPOSE_OF_TRANSFER: &str = "TRANSFER"; pub(crate) use builder::{CashNoteBuilder, TransactionBuilder}; pub(crate) use transaction::Input; diff --git a/sn_transfers/src/genesis.rs b/sn_transfers/src/genesis.rs index c40e8b8435..acdcce290d 100644 --- a/sn_transfers/src/genesis.rs +++ b/sn_transfers/src/genesis.rs @@ -8,7 +8,7 @@ use super::wallet::HotWallet; -use crate::cashnotes::CASH_NOTE_PURPOSE_FOR_GENESIS; +use crate::cashnotes::CASHNOTE_PURPOSE_OF_GENESIS; use crate::{ CashNote, DerivationIndex, Hash, Input, MainPubkey, MainSecretKey, NanoTokens, SignedSpend, Transaction, TransactionBuilder, TransferError as CashNoteError, @@ -160,7 +160,7 @@ pub fn create_first_cash_note_from_key( ) .add_output( NanoTokens::from(GENESIS_CASHNOTE_AMOUNT), - CASH_NOTE_PURPOSE_FOR_GENESIS.to_string(), + CASHNOTE_PURPOSE_OF_GENESIS.to_string(), main_pubkey, derivation_index, ) diff --git a/sn_transfers/src/lib.rs b/sn_transfers/src/lib.rs index 474820bf30..1bb42fd573 100644 --- a/sn_transfers/src/lib.rs +++ b/sn_transfers/src/lib.rs @@ -21,8 +21,8 @@ pub(crate) use cashnotes::{Input, TransactionBuilder}; pub use cashnotes::{ CashNote, CashNoteOutputDetails, DerivationIndex, DerivedSecretKey, Hash, MainPubkey, MainSecretKey, NanoTokens, SignedSpend, Spend, SpendAddress, Transaction, UniquePubkey, - UnsignedTransfer, CASH_NOTE_PURPOSE_FOR_GENESIS, CASH_NOTE_PURPOSE_FOR_NETWORK_ROYALTIES, - CASH_NOTE_PURPOSE_FOR_TRANSFER, + UnsignedTransfer, CASHNOTE_PURPOSE_OF_GENESIS, CASHNOTE_PURPOSE_OF_NETWORK_ROYALTIES, + CASHNOTE_PURPOSE_OF_TRANSFER, }; pub use error::{Result, TransferError}; pub use transfers::{CashNoteRedemption, OfflineTransfer, Transfer}; diff --git a/sn_transfers/src/transfers/offline_transfer.rs b/sn_transfers/src/transfers/offline_transfer.rs index 6551768fa4..4899bf4736 100644 --- a/sn_transfers/src/transfers/offline_transfer.rs +++ b/sn_transfers/src/transfers/offline_transfer.rs @@ -7,7 +7,7 @@ // permissions and limitations relating to use of the SAFE Network Software. use crate::{ - cashnotes::{CashNoteBuilder, UnsignedTransfer, CASH_NOTE_PURPOSE_FOR_CHANGE}, + cashnotes::{CashNoteBuilder, UnsignedTransfer, CASHNOTE_PURPOSE_OF_CHANGE}, rng, CashNote, CashNoteOutputDetails, DerivationIndex, DerivedSecretKey, Hash, Input, MainPubkey, NanoTokens, Result, SignedSpend, Transaction, TransactionBuilder, TransferError, UniquePubkey, NETWORK_ROYALTIES_PK, @@ -277,7 +277,7 @@ fn create_transaction_builder_with( if !change.is_zero() { tx_builder = tx_builder.add_output( change, - CASH_NOTE_PURPOSE_FOR_CHANGE.to_string(), + CASHNOTE_PURPOSE_OF_CHANGE.to_string(), change_to, derivation_index, ); diff --git a/sn_transfers/src/wallet/hot_wallet.rs b/sn_transfers/src/wallet/hot_wallet.rs index 9be2454e6a..60846f0a58 100644 --- a/sn_transfers/src/wallet/hot_wallet.rs +++ b/sn_transfers/src/wallet/hot_wallet.rs @@ -24,7 +24,7 @@ use crate::{ transfers::{CashNotesAndSecretKey, OfflineTransfer}, CashNote, CashNoteOutputDetails, CashNoteRedemption, DerivationIndex, DerivedSecretKey, Hash, MainPubkey, MainSecretKey, NanoTokens, SignedSpend, Spend, Transaction, Transfer, UniquePubkey, - WalletError, CASH_NOTE_PURPOSE_FOR_NETWORK_ROYALTIES, NETWORK_ROYALTIES_PK, + WalletError, CASHNOTE_PURPOSE_OF_NETWORK_ROYALTIES, NETWORK_ROYALTIES_PK, }; use std::{ collections::{BTreeMap, BTreeSet, HashSet}, @@ -402,7 +402,7 @@ impl HotWallet { let royalties_fee = calculate_royalties_fee(quote.cost); let royalties_payee = ( royalties_fee, - CASH_NOTE_PURPOSE_FOR_NETWORK_ROYALTIES.to_string(), + CASHNOTE_PURPOSE_OF_NETWORK_ROYALTIES.to_string(), *NETWORK_ROYALTIES_PK, DerivationIndex::random(&mut rng), ); From d884f9da4769795c6abc4c0aece59fc45462a5a8 Mon Sep 17 00:00:00 2001 From: qima Date: Thu, 2 May 2024 19:25:00 +0800 Subject: [PATCH 203/205] chore(CI): disable the audit step during memcheck test --- .github/workflows/memcheck.yml | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/.github/workflows/memcheck.yml b/.github/workflows/memcheck.yml index b895dae3e7..528ba95adb 100644 --- a/.github/workflows/memcheck.yml +++ b/.github/workflows/memcheck.yml @@ -279,14 +279,15 @@ jobs: SN_LOG: "all" timeout-minutes: 10 - - name: Audit from genesis to collect entire spend DAG and dump to a dot file - run: | - cargo run --bin safe --release -- --log-output-dest=data-dir wallet audit --dot > spend_dag_and_statistics.txt - echo "==============================================================================" - cat spend_dag_and_statistics.txt - env: - SN_LOG: "all" - timeout-minutes: 60 + # Disable this test temporarily as it takes too long time, which blocks the merge queue. + # - name: Audit from genesis to collect entire spend DAG and dump to a dot file + # run: | + # cargo run --bin safe --release -- --log-output-dest=data-dir wallet audit --dot > spend_dag_and_statistics.txt + # echo "==============================================================================" + # cat spend_dag_and_statistics.txt + # env: + # SN_LOG: "all" + # timeout-minutes: 60 - name: Check nodes running shell: bash @@ -446,10 +447,11 @@ jobs: continue-on-error: true if: always() - - name: Upload spend DAG and statistics - uses: actions/upload-artifact@main - with: - name: memory_check_spend_dag_and_statistics - path: spend_dag_and_statistics.txt - continue-on-error: true - if: always() + # Re-enable this block once the previous step of audit got enabled. + # - name: Upload spend DAG and statistics + # uses: actions/upload-artifact@main + # with: + # name: memory_check_spend_dag_and_statistics + # path: spend_dag_and_statistics.txt + # continue-on-error: true + # if: always() From 36ac059e4a41cf3cd9d4fb5750e588c9855b764a Mon Sep 17 00:00:00 2001 From: Roland Sherwin Date: Thu, 2 May 2024 18:39:02 +0530 Subject: [PATCH 204/205] chore: disable network contacts feature flag on launchpad --- sn_node_launchpad/Cargo.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sn_node_launchpad/Cargo.toml b/sn_node_launchpad/Cargo.toml index 71073ffa90..a1392d7396 100644 --- a/sn_node_launchpad/Cargo.toml +++ b/sn_node_launchpad/Cargo.toml @@ -15,9 +15,7 @@ name = "node-launchpad" path = "src/main.rs" [dependencies] -sn_peers_acquisition = { verison = "0.2.10", path = "../sn_peers_acquisition", features = [ - "network-contacts", -] } +sn_peers_acquisition = { verison = "0.2.10", path = "../sn_peers_acquisition" } sn_service_management = { verison = "0.2.4", path = "../sn_service_management" } better-panic = "0.3.0" clap = { version = "4.4.5", features = [ From d1c0cc1295f5a974055e2de423de7e8647285bca Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Thu, 2 May 2024 15:18:05 +0100 Subject: [PATCH 205/205] fix: do not create wallet on registry refresh Use the `try_load_from` mechanism, rather than `try_load`, because the former will not create the wallet directory if it does not exist. The node manager refreshes its registry when it runs commands like `start` and `stop`, which can be running as the root user. The refresh now involves getting the balance of the wallet. Therefore, when `start` ran for the first time, the `try_load` function was creating a directory owned by the root user. This would cause the node to crash because it couldn't write to that directory. A new Clippy warning is also fixed and I removed the superfluous comments around the code it was referring to. --- sn_logging/src/lib.rs | 9 +++------ sn_node_manager/src/add_services/mod.rs | 4 +++- sn_node_manager/src/lib.rs | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/sn_logging/src/lib.rs b/sn_logging/src/lib.rs index 39a4c868e7..7778507b9f 100644 --- a/sn_logging/src/lib.rs +++ b/sn_logging/src/lib.rs @@ -37,16 +37,13 @@ fn current_exe_name() -> String { .next() .and_then(|arg| { std::path::Path::new(&arg).file_name().map(|s| { - let mut name = s.to_string_lossy().into_owned(); - // remove sn_ prefix if present - name = name.strip_prefix("sn_").unwrap_or(&name).to_owned(); + let mut name = s.to_string_lossy().to_string(); + name = name.strip_prefix("sn_").unwrap_or(&name).to_string(); - // remove .exe prefix on windows if cfg!(windows) && name.to_lowercase().ends_with(".exe") { - name = name.strip_suffix(".exe").unwrap_or(&name).to_owned(); + name = name.strip_suffix(".exe").unwrap_or(&name).to_string(); } - // if the name is safe, identify it is the client if name == "safe" { name = "client".to_string(); } diff --git a/sn_node_manager/src/add_services/mod.rs b/sn_node_manager/src/add_services/mod.rs index 39e0b110dd..83e81eaeca 100644 --- a/sn_node_manager/src/add_services/mod.rs +++ b/sn_node_manager/src/add_services/mod.rs @@ -96,7 +96,9 @@ pub async fn add_node( } if options.env_variables.is_some() { - node_registry.environment_variables = options.env_variables.clone(); + node_registry + .environment_variables + .clone_from(&options.env_variables); should_save = true; } diff --git a/sn_node_manager/src/lib.rs b/sn_node_manager/src/lib.rs index a3a01320f6..233a9d0cc5 100644 --- a/sn_node_manager/src/lib.rs +++ b/sn_node_manager/src/lib.rs @@ -429,7 +429,7 @@ pub async fn refresh_node_registry( for node in &mut node_registry.nodes { // The `status` command can run before a node is started and therefore before its wallet // exists. - match HotWallet::load_from(&node.data_dir_path) { + match HotWallet::try_load_from(&node.data_dir_path) { Ok(wallet) => node.reward_balance = Some(wallet.balance()), Err(_) => node.reward_balance = None, }