diff --git a/Cargo.lock b/Cargo.lock index 6e6ec97b7f..b2f76027d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -723,7 +723,7 @@ dependencies = [ ] [[package]] -name = "ant-bootstrap-cache" +name = "ant-bootstrap" version = "0.1.0" dependencies = [ "ant-logging", @@ -851,6 +851,7 @@ name = "ant-networking" version = "0.19.5" dependencies = [ "aes-gcm-siv", + "ant-bootstrap", "ant-build-info", "ant-evm", "ant-protocol", @@ -897,6 +898,7 @@ dependencies = [ name = "ant-node" version = "0.112.6" dependencies = [ + "ant-bootstrap", "ant-build-info", "ant-evm", "ant-logging", diff --git a/Cargo.toml b/Cargo.toml index da1073ed31..eeafdece63 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace] resolver = "2" members = [ - "ant-bootstrap-cache", + "ant-bootstrap", "ant-build-info", "ant-cli", "ant-evm", diff --git a/ant-bootstrap-cache/Cargo.toml b/ant-bootstrap/Cargo.toml similarity index 91% rename from ant-bootstrap-cache/Cargo.toml rename to ant-bootstrap/Cargo.toml index 593126b942..c736f241a6 100644 --- a/ant-bootstrap-cache/Cargo.toml +++ b/ant-bootstrap/Cargo.toml @@ -1,10 +1,10 @@ [package] authors = ["MaidSafe Developers "] -description = "Bootstrap Cache functionality for Autonomi" +description = "Bootstrap functionality for Autonomi" edition = "2021" homepage = "https://maidsafe.net" license = "GPL-3.0" -name = "ant-bootstrap-cache" +name = "ant-bootstrap" readme = "README.md" repository = "https://github.com/maidsafe/autonomi" version = "0.1.0" diff --git a/ant-bootstrap-cache/README.md b/ant-bootstrap/README.md similarity index 100% rename from ant-bootstrap-cache/README.md rename to ant-bootstrap/README.md diff --git a/ant-bootstrap-cache/src/cache_store.rs b/ant-bootstrap/src/cache_store.rs similarity index 72% rename from ant-bootstrap-cache/src/cache_store.rs rename to ant-bootstrap/src/cache_store.rs index 39e14e6928..1d022307cc 100644 --- a/ant-bootstrap-cache/src/cache_store.rs +++ b/ant-bootstrap/src/cache_store.rs @@ -7,8 +7,8 @@ // permissions and limitations relating to use of the SAFE Network Software. use crate::{ - craft_valid_multiaddr, multiaddr_get_peer_id, BootstrapAddr, BootstrapAddresses, - BootstrapConfig, Error, InitialPeerDiscovery, Result, + craft_valid_multiaddr, initial_peers::PeersArgs, multiaddr_get_peer_id, BootstrapAddr, + BootstrapAddresses, BootstrapCacheConfig, Error, Result, }; use fs2::FileExt; use libp2p::multiaddr::Protocol; @@ -24,7 +24,7 @@ use tempfile::NamedTempFile; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CacheData { - peers: std::collections::HashMap, + pub(crate) peers: std::collections::HashMap, #[serde(default = "SystemTime::now")] last_updated: SystemTime, #[serde(default = "default_version")] @@ -69,7 +69,7 @@ impl CacheData { /// - Removes all peers with empty addrs set /// - Maintains `max_addr` per peer by removing the addr with the lowest success rate /// - Maintains `max_peers` in the list by removing the peer with the oldest last_seen - pub fn perform_cleanup(&mut self, cfg: &BootstrapConfig) { + pub fn perform_cleanup(&mut self, cfg: &BootstrapCacheConfig) { self.peers.values_mut().for_each(|bootstrap_addresses| { bootstrap_addresses.0.retain(|bootstrap_addr| { let now = SystemTime::now(); @@ -100,7 +100,7 @@ impl CacheData { } /// Remove the oldest peers until we're under the max_peers limit - pub fn try_remove_oldest_peers(&mut self, cfg: &BootstrapConfig) { + pub fn try_remove_oldest_peers(&mut self, cfg: &BootstrapCacheConfig) { if self.peers.len() > cfg.max_peers { let mut peer_last_seen_map = HashMap::new(); for (peer, addrs) in self.peers.iter() { @@ -149,48 +149,21 @@ impl Default for CacheData { #[derive(Clone, Debug)] pub struct BootstrapCacheStore { - cache_path: PathBuf, - config: BootstrapConfig, - data: CacheData, + pub(crate) cache_path: PathBuf, + pub(crate) config: BootstrapCacheConfig, + pub(crate) data: CacheData, /// This is our last known state of the cache on disk, which is shared across all instances. /// This is not updated until `sync_to_disk` is called. - old_shared_state: CacheData, + pub(crate) old_shared_state: CacheData, } impl BootstrapCacheStore { - pub fn config(&self) -> &BootstrapConfig { + pub fn config(&self) -> &BootstrapCacheConfig { &self.config } - pub async fn new(config: BootstrapConfig) -> Result { - info!("Creating new CacheStore with config: {:?}", config); - let cache_path = config.cache_file_path.clone(); - - // Create cache directory if it doesn't exist - if let Some(parent) = cache_path.parent() { - if !parent.exists() { - info!("Attempting to create cache directory at {parent:?}"); - fs::create_dir_all(parent).inspect_err(|err| { - warn!("Failed to create cache directory at {parent:?}: {err}"); - })?; - } - } - - let mut store = Self { - cache_path, - config, - data: CacheData::default(), - old_shared_state: CacheData::default(), - }; - - store.init().await?; - - info!("Successfully created CacheStore and initialized it."); - - Ok(store) - } - - pub async fn new_without_init(config: BootstrapConfig) -> Result { + /// Create a empty CacheStore with the given configuration + pub fn empty(config: BootstrapCacheConfig) -> Result { info!("Creating new CacheStore with config: {:?}", config); let cache_path = config.cache_file_path.clone(); @@ -211,146 +184,26 @@ impl BootstrapCacheStore { old_shared_state: CacheData::default(), }; - info!("Successfully created CacheStore without initializing the data."); Ok(store) } - pub async fn init(&mut self) -> Result<()> { - let data = if self.cache_path.exists() { - info!( - "Cache file exists at {:?}, attempting to load", - self.cache_path - ); - match Self::load_cache_data(&self.config).await { - Ok(data) => { - info!( - "Successfully loaded cache data with {} peers", - data.peers.len() - ); - // If cache data exists but has no peers and file is not read-only, - // fallback to default - let is_readonly = self - .cache_path - .metadata() - .map(|m| m.permissions().readonly()) - .unwrap_or(false); - - if data.peers.is_empty() && !is_readonly { - info!("Cache is empty and not read-only, falling back to default"); - Self::fallback_to_default(&self.config).await? - } else { - // Ensure we don't exceed max_peers - let mut filtered_data = data; - if filtered_data.peers.len() > self.config.max_peers { - info!( - "Trimming cache from {} to {} peers", - filtered_data.peers.len(), - self.config.max_peers - ); - - filtered_data.peers = filtered_data - .peers - .into_iter() - .take(self.config.max_peers) - .collect(); - } - filtered_data - } - } - Err(e) => { - warn!("Failed to load cache data: {}", e); - // If we can't read or parse the cache file, fallback to default - Self::fallback_to_default(&self.config).await? - } - } - } else { - info!( - "Cache file does not exist at {:?}, falling back to default", - self.cache_path - ); - // If cache file doesn't exist, fallback to default - Self::fallback_to_default(&self.config).await? - }; - - // Update the store's data - self.data = data.clone(); - self.old_shared_state = data; - - // Save the default data to disk - self.sync_and_save_to_disk(false).await?; - + pub async fn initialize_from_peers_arg(&mut self, peers_arg: &PeersArgs) -> Result<()> { + peers_arg + .get_bootstrap_addr_and_initialize_cache(Some(self)) + .await?; + self.sync_and_save_to_disk(true).await?; Ok(()) } - async fn fallback_to_default(config: &BootstrapConfig) -> Result { - info!("Falling back to default peers from endpoints"); - let mut data = CacheData { - peers: std::collections::HashMap::new(), - last_updated: SystemTime::now(), - version: default_version(), - }; - - // If no endpoints are configured, just return empty cache - if config.endpoints.is_empty() { - warn!("No endpoints configured, returning empty cache"); - return Ok(data); - } - - // Try to discover peers from configured endpoints - let discovery = InitialPeerDiscovery::with_endpoints(config.endpoints.clone())?; - match discovery.fetch_bootstrap_addresses().await { - Ok(addrs) => { - info!("Successfully fetched {} peers from endpoints", addrs.len()); - // Only add up to max_peers from the discovered peers - let mut count = 0; - for bootstrap_addr in addrs.into_iter() { - if count >= config.max_peers { - break; - } - if let Some(peer_id) = bootstrap_addr.peer_id() { - data.insert(peer_id, bootstrap_addr); - count += 1; - } - } - - // Create parent directory if it doesn't exist - if let Some(parent) = config.cache_file_path.parent() { - if !parent.exists() { - info!("Creating cache directory at {:?}", parent); - if let Err(e) = fs::create_dir_all(parent) { - warn!("Failed to create cache directory: {}", e); - } - } - } - - // Try to write the cache file immediately - match serde_json::to_string_pretty(&data) { - Ok(json) => { - info!("Writing {} peers to cache file", data.peers.len()); - if let Err(e) = fs::write(&config.cache_file_path, json) { - warn!("Failed to write cache file: {}", e); - } else { - info!( - "Successfully wrote cache file at {:?}", - config.cache_file_path - ); - } - } - Err(e) => { - warn!("Failed to serialize cache data: {}", e); - } - } - - Ok(data) - } - Err(e) => { - warn!("Failed to fetch peers from endpoints: {}", e); - Ok(data) // Return empty cache on error - } - } + pub async fn initialize_from_local_cache(&mut self) -> Result<()> { + self.data = Self::load_cache_data(&self.config).await?; + self.old_shared_state = self.data.clone(); + Ok(()) } - async fn load_cache_data(cfg: &BootstrapConfig) -> Result { + /// Load cache data from disk + /// Make sure to have clean addrs inside the cache as we don't call craft_valid_multiaddr + pub async fn load_cache_data(cfg: &BootstrapCacheConfig) -> Result { // Try to open the file with read permissions let mut file = match OpenOptions::new().read(true).open(&cfg.cache_file_path) { Ok(f) => f, @@ -556,6 +409,7 @@ impl BootstrapCacheStore { } async fn atomic_write(&self) -> Result<()> { + info!("Writing cache to disk: {:?}", self.cache_path); // Create parent directory if it doesn't exist if let Some(parent) = self.cache_path.parent() { fs::create_dir_all(parent).map_err(Error::from)?; @@ -583,6 +437,8 @@ impl BootstrapCacheStore { error!("Failed to persist file with err: {err:?}"); })?; + info!("Cache written to disk: {:?}", self.cache_path); + // Lock will be automatically released when file is dropped Ok(()) } @@ -597,11 +453,9 @@ mod tests { let temp_dir = tempdir().unwrap(); let cache_file = temp_dir.path().join("cache.json"); - let config = crate::BootstrapConfig::empty() - .unwrap() - .with_cache_path(&cache_file); + let config = crate::BootstrapCacheConfig::empty().with_cache_path(&cache_file); - let store = BootstrapCacheStore::new(config).await.unwrap(); + let store = BootstrapCacheStore::empty(config).unwrap(); (store.clone(), store.cache_path.clone()) } diff --git a/ant-bootstrap-cache/src/config.rs b/ant-bootstrap/src/config.rs similarity index 77% rename from ant-bootstrap-cache/src/config.rs rename to ant-bootstrap/src/config.rs index e02fa8a590..52d85b7dee 100644 --- a/ant-bootstrap-cache/src/config.rs +++ b/ant-bootstrap/src/config.rs @@ -12,7 +12,6 @@ use std::{ path::{Path, PathBuf}, time::Duration, }; -use url::Url; /// The duration since last)seen before removing the address of a Peer. const ADDR_EXPIRY_DURATION: Duration = Duration::from_secs(24 * 60 * 60); // 24 hours @@ -31,11 +30,9 @@ const MAX_BOOTSTRAP_CACHE_SAVE_INTERVAL: Duration = Duration::from_secs(24 * 60 /// Configuration for the bootstrap cache #[derive(Clone, Debug)] -pub struct BootstrapConfig { +pub struct BootstrapCacheConfig { /// The duration since last)seen before removing the address of a Peer. pub addr_expiry_duration: Duration, - /// List of bootstrap endpoints to fetch peer information from - pub endpoints: Vec, /// Maximum number of peers to keep in the cache pub max_peers: usize, /// Maximum number of addresses stored per peer. @@ -52,19 +49,11 @@ pub struct BootstrapConfig { pub cache_save_scaling_factor: u64, } -impl BootstrapConfig { +impl BootstrapCacheConfig { /// Creates a new BootstrapConfig with default settings pub fn default_config() -> Result { Ok(Self { addr_expiry_duration: ADDR_EXPIRY_DURATION, - endpoints: vec![ - "https://sn-testnet.s3.eu-west-2.amazonaws.com/bootstrap_cache.json" - .parse() - .expect("Failed to parse URL"), - "https://sn-testnet.s3.eu-west-2.amazonaws.com/network-contacts" - .parse() - .expect("Failed to parse URL"), - ], max_peers: MAX_PEERS, max_addrs_per_peer: MAX_ADDRS_PER_PEER, cache_file_path: default_cache_path()?, @@ -76,18 +65,17 @@ impl BootstrapConfig { } /// Creates a new BootstrapConfig with empty settings - pub fn empty() -> Result { - Ok(Self { + pub fn empty() -> Self { + Self { addr_expiry_duration: ADDR_EXPIRY_DURATION, - endpoints: vec![], max_peers: MAX_PEERS, max_addrs_per_peer: MAX_ADDRS_PER_PEER, - cache_file_path: default_cache_path()?, + cache_file_path: PathBuf::new(), disable_cache_writing: false, min_cache_save_duration: MIN_BOOTSTRAP_CACHE_SAVE_INTERVAL, max_cache_save_duration: MAX_BOOTSTRAP_CACHE_SAVE_INTERVAL, cache_save_scaling_factor: 2, - }) + } } /// Set a new addr expiry duration @@ -96,25 +84,6 @@ impl BootstrapConfig { self } - /// Update the config with custom endpoints - pub fn with_endpoints(mut self, endpoints: Vec) -> Self { - self.endpoints = endpoints; - self - } - - /// Update the config with default endpoints - pub fn with_default_endpoints(mut self) -> Self { - self.endpoints = vec![ - "https://sn-testnet.s3.eu-west-2.amazonaws.com/bootstrap_cache.json" - .parse() - .expect("Failed to parse URL"), - "https://sn-testnet.s3.eu-west-2.amazonaws.com/network-contacts" - .parse() - .expect("Failed to parse URL"), - ]; - self - } - /// Update the config with a custom cache file path pub fn with_cache_path>(mut self, path: P) -> Self { self.cache_file_path = path.as_ref().to_path_buf(); diff --git a/ant-bootstrap-cache/src/initial_peer_discovery.rs b/ant-bootstrap/src/contacts.rs similarity index 100% rename from ant-bootstrap-cache/src/initial_peer_discovery.rs rename to ant-bootstrap/src/contacts.rs diff --git a/ant-bootstrap-cache/src/error.rs b/ant-bootstrap/src/error.rs similarity index 95% rename from ant-bootstrap-cache/src/error.rs rename to ant-bootstrap/src/error.rs index 92bb997d63..9946d4680f 100644 --- a/ant-bootstrap-cache/src/error.rs +++ b/ant-bootstrap/src/error.rs @@ -10,6 +10,8 @@ use thiserror::Error; #[derive(Debug, Error)] pub enum Error { + #[error("Failed to obtain any bootstrap peers")] + NoBootstrapPeersFound, #[error("Failed to parse cache data")] FailedToParseCacheData, #[error("Could not obtain data directory")] diff --git a/ant-bootstrap/src/initial_peers.rs b/ant-bootstrap/src/initial_peers.rs new file mode 100644 index 0000000000..de49d9f006 --- /dev/null +++ b/ant-bootstrap/src/initial_peers.rs @@ -0,0 +1,202 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use crate::{ + craft_valid_multiaddr, + error::{Error, Result}, + BootstrapAddr, BootstrapCacheConfig, BootstrapCacheStore, InitialPeerDiscovery, +}; +use clap::Args; +use libp2p::Multiaddr; +use url::Url; + +/// Command line arguments for peer configuration +#[derive(Args, Debug, Clone, Default)] +pub struct PeersArgs { + /// Set to indicate this is the first node in a new network + /// + /// If this argument is used, any others will be ignored because they do not apply to the first + /// node. + #[clap(long)] + pub first: bool, + /// Addr(s) to use for bootstrap, in a 'multiaddr' format containing the peer ID. + /// + /// A multiaddr looks like + /// '/ip4/1.2.3.4/tcp/1200/tcp/p2p/12D3KooWRi6wF7yxWLuPSNskXc6kQ5cJ6eaymeMbCRdTnMesPgFx' where + /// `1.2.3.4` is the IP, `1200` is the port and the (optional) last part is the peer ID. + /// + /// This argument can be provided multiple times to connect to multiple peers. + /// + /// Alternatively, the `ANT_PEERS` environment variable can provide a comma-separated peer + /// list. + #[clap( + long = "peer", + env = "ANT_PEERS", + value_name = "multiaddr", + value_delimiter = ',', + conflicts_with = "first" + )] + pub addrs: Vec, + /// URL to fetch network contacts from + pub network_contacts_url: Option, + /// Use only local discovery (mDNS) + pub local: bool, + /// Connect to a testnet. This disables fetching peers from network contacts. + pub testnet: bool, +} + +impl PeersArgs { + /// Get bootstrap peers + /// Order of precedence: + /// 1. Addresses from arguments + /// 2. Addresses from environment variable SAFE_PEERS + /// 3. Addresses from cache + /// 4. Addresses from network contacts URL + pub async fn get_bootstrap_addr(&self) -> Result> { + self.get_bootstrap_addr_and_initialize_cache(None).await + } + + pub async fn get_addrs(&self) -> Result> { + Ok(self + .get_bootstrap_addr() + .await? + .into_iter() + .map(|addr| addr.addr) + .collect()) + } + + /// Helper function to fetch bootstrap addresses and initialize cache based on the passed in args. + pub(crate) async fn get_bootstrap_addr_and_initialize_cache( + &self, + mut cache: Option<&mut BootstrapCacheStore>, + ) -> Result> { + // If this is the first node, return an empty list + if self.first { + info!("First node in network, no initial bootstrap peers"); + if let Some(cache) = cache { + info!("Clearing cache for 'first' node"); + cache.clear_peers_and_save().await?; + } + return Ok(vec![]); + } + + // If local mode is enabled, return empty store (will use mDNS) + if self.local { + info!("Local mode enabled, using only local discovery."); + if let Some(cache) = cache { + info!("Setting config to not write to cache, as 'local' mode is enabled"); + cache.config.disable_cache_writing = true; + } + return Ok(vec![]); + } + + let mut bootstrap_addresses = vec![]; + + // Add addrs from environment variable if present + if let Ok(env_string) = std::env::var("SAFE_PEERS") { + for multiaddr_str in env_string.split(',') { + if let Ok(addr) = multiaddr_str.parse() { + if let Some(addr) = craft_valid_multiaddr(&addr) { + info!("Adding addr from environment: {addr}",); + bootstrap_addresses.push(BootstrapAddr::new(addr)); + } else { + warn!("Invalid peer address format from environment: {addr}",); + } + } + } + } + + // Add addrs from arguments if present + for addr in &self.addrs { + if let Some(addr) = craft_valid_multiaddr(addr) { + info!("Adding addr from arguments: {addr}"); + bootstrap_addresses.push(BootstrapAddr::new(addr)); + } else { + warn!("Invalid multiaddress format from arguments: {addr}"); + } + } + + // If we have a network contacts URL, fetch addrs from there. + if let Some(url) = self.network_contacts_url.clone() { + info!("Fetching bootstrap address from network contacts URL: {url}",); + let peer_discovery = InitialPeerDiscovery::with_endpoints(vec![url])?; + let addrs = peer_discovery.fetch_bootstrap_addresses().await?; + bootstrap_addresses.extend(addrs); + } + + // Return here if we fetched peers from the args + if !bootstrap_addresses.is_empty() { + if let Some(cache) = cache.as_mut() { + info!("Initializing cache with bootstrap addresses from arguments"); + for addr in &bootstrap_addresses { + cache.add_addr(addr.addr.clone()); + } + } + return Ok(bootstrap_addresses); + } + + // load from cache if present + let cfg = if let Some(cache) = cache.as_ref() { + Some(cache.config.clone()) + } else { + BootstrapCacheConfig::default_config().ok() + }; + if let Some(cfg) = cfg { + info!("Loading bootstrap addresses from cache"); + if let Ok(data) = BootstrapCacheStore::load_cache_data(&cfg).await { + if let Some(cache) = cache.as_mut() { + info!("Initializing cache with bootstrap addresses from cache"); + cache.data = data.clone(); + cache.old_shared_state = data.clone(); + } + + bootstrap_addresses = data + .peers + .into_iter() + .filter_map(|(_, addrs)| { + addrs + .0 + .into_iter() + .min_by_key(|addr| addr.failure_rate() as u64) + }) + .collect(); + } + } + + if !bootstrap_addresses.is_empty() { + return Ok(bootstrap_addresses); + } + + if !self.testnet { + let mainnet_contact = vec![ + "https://sn-testnet.s3.eu-west-2.amazonaws.com/bootstrap_cache.json" + .parse() + .expect("Failed to parse URL"), + "https://sn-testnet.s3.eu-west-2.amazonaws.com/network-contacts" + .parse() + .expect("Failed to parse URL"), + ]; + let peer_discovery = InitialPeerDiscovery::with_endpoints(mainnet_contact)?; + let addrs = peer_discovery.fetch_bootstrap_addresses().await?; + if let Some(cache) = cache.as_mut() { + info!("Initializing cache with bootstrap addresses from mainnet contacts"); + for addr in addrs.iter() { + cache.add_addr(addr.addr.clone()); + } + } + bootstrap_addresses = addrs; + } + + if !bootstrap_addresses.is_empty() { + Ok(bootstrap_addresses) + } else { + error!("No initial bootstrap peers found through any means"); + Err(Error::NoBootstrapPeersFound) + } + } +} diff --git a/ant-bootstrap-cache/src/lib.rs b/ant-bootstrap/src/lib.rs similarity index 72% rename from ant-bootstrap-cache/src/lib.rs rename to ant-bootstrap/src/lib.rs index 37caedd3bd..fef4b5c1ef 100644 --- a/ant-bootstrap-cache/src/lib.rs +++ b/ant-bootstrap/src/lib.rs @@ -21,19 +21,21 @@ //! # Example //! //! ```no_run -//! use ant_bootstrap_cache::{BootstrapCacheStore, BootstrapConfig, PeersArgs}; +//! use ant_bootstrap::{BootstrapCacheStore, BootstrapCacheConfig, PeersArgs}; //! use url::Url; //! //! # async fn example() -> Result<(), Box> { -//! let config = BootstrapConfig::empty().unwrap(); +//! let config = BootstrapCacheConfig::empty(); //! let args = PeersArgs { //! first: false, //! addrs: vec![], //! network_contacts_url: Some(Url::parse("https://example.com/peers")?), //! local: false, +//! testnet: false, //! }; //! -//! let store = BootstrapCacheStore::from_args(args, config).await?; +//! let mut store = BootstrapCacheStore::empty(config)?; +//! store.initialize_from_peers_arg(&args).await?; //! let addrs = store.get_addrs(); //! # Ok(()) //! # } @@ -44,19 +46,20 @@ extern crate tracing; mod cache_store; pub mod config; +pub mod contacts; mod error; -mod initial_peer_discovery; +mod initial_peers; use libp2p::{multiaddr::Protocol, Multiaddr, PeerId}; use serde::{Deserialize, Serialize}; use std::time::SystemTime; use thiserror::Error; -use url::Url; pub use cache_store::BootstrapCacheStore; -pub use config::BootstrapConfig; +pub use config::BootstrapCacheConfig; +pub use contacts::InitialPeerDiscovery; pub use error::{Error, Result}; -pub use initial_peer_discovery::InitialPeerDiscovery; +pub use initial_peers::PeersArgs; /// Structure representing a list of bootstrap endpoints #[derive(Debug, Clone, Serialize, Deserialize)] @@ -254,95 +257,6 @@ impl BootstrapAddr { } } -/// Command line arguments for peer configuration -#[derive(Debug, Clone, Default)] -pub struct PeersArgs { - /// First node in the network - pub first: bool, - /// List of addresses - pub addrs: Vec, - /// URL to fetch network contacts from - pub network_contacts_url: Option, - /// Use only local discovery (mDNS) - pub local: bool, -} - -impl BootstrapCacheStore { - /// Create a new CacheStore from command line arguments - /// This also initializes the store with the provided bootstrap addresses - pub async fn from_args(args: PeersArgs, mut config: BootstrapConfig) -> Result { - if let Some(url) = &args.network_contacts_url { - config.endpoints.push(url.clone()); - } - - // If this is the first node, return empty store with no fallback - if args.first { - info!("First node in network, returning empty store"); - let mut store = Self::new_without_init(config).await?; - store.clear_peers_and_save().await?; - return Ok(store); - } - - // If local mode is enabled, return empty store (will use mDNS) - if args.local { - info!("Local mode enabled, using only local discovery. Cache writing is disabled"); - config.disable_cache_writing = true; - let store = Self::new_without_init(config).await?; - return Ok(store); - } - - // Create a new store but don't load from cache or fetch from endpoints yet - let mut store = Self::new_without_init(config).await?; - - // Add addrs from environment variable if present - if let Ok(env_string) = std::env::var("SAFE_PEERS") { - for multiaddr_str in env_string.split(',') { - if let Ok(addr) = multiaddr_str.parse() { - if let Some(addr) = craft_valid_multiaddr(&addr) { - info!("Adding addr from environment: {addr}",); - store.add_addr(addr); - } else { - warn!("Invalid peer address format from environment: {}", addr); - } - } - } - } - - // Add addrs from arguments if present - for addr in args.addrs { - if let Some(addr) = craft_valid_multiaddr(&addr) { - info!("Adding addr from arguments: {addr}"); - store.add_addr(addr); - } else { - warn!("Invalid multiaddress format from arguments: {addr}"); - } - } - - // If we have a network contacts URL, fetch addrs from there. - if let Some(url) = args.network_contacts_url { - info!( - "Fetching bootstrap address from network contacts URL: {}", - url - ); - let peer_discovery = InitialPeerDiscovery::with_endpoints(vec![url])?; - let bootstrap_addresses = peer_discovery.fetch_bootstrap_addresses().await?; - for addr in bootstrap_addresses { - store.add_addr(addr.addr); - } - } - - // If we have peers, update cache and return, else initialize from cache - if store.peer_count() > 0 { - info!("Using provided peers and updating cache"); - store.sync_and_save_to_disk(false).await?; - } else { - store.init().await?; - } - - Ok(store) - } -} - /// Craft a proper address to avoid any ill formed addresses pub fn craft_valid_multiaddr(addr: &Multiaddr) -> Option { let peer_id = addr diff --git a/ant-bootstrap-cache/tests/address_format_tests.rs b/ant-bootstrap/tests/address_format_tests.rs similarity index 71% rename from ant-bootstrap-cache/tests/address_format_tests.rs rename to ant-bootstrap/tests/address_format_tests.rs index 73f8856465..369fcab68a 100644 --- a/ant-bootstrap-cache/tests/address_format_tests.rs +++ b/ant-bootstrap/tests/address_format_tests.rs @@ -6,7 +6,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use ant_bootstrap_cache::{BootstrapCacheStore, BootstrapConfig, PeersArgs}; +use ant_bootstrap::{BootstrapCacheConfig, BootstrapCacheStore, PeersArgs}; use ant_logging::LogBuilder; use libp2p::Multiaddr; use tempfile::TempDir; @@ -16,12 +16,11 @@ use wiremock::{ }; // Setup function to create a new temp directory and config for each test -async fn setup() -> (TempDir, BootstrapConfig) { +async fn setup() -> (TempDir, BootstrapCacheConfig) { let temp_dir = TempDir::new().unwrap(); let cache_path = temp_dir.path().join("cache.json"); - let config = BootstrapConfig::empty() - .unwrap() + let config = BootstrapCacheConfig::empty() .with_cache_path(&cache_path) .with_max_peers(50); @@ -48,9 +47,11 @@ async fn test_multiaddr_format_parsing() -> Result<(), Box>(); assert_eq!(bootstrap_addresses.len(), 1, "Should have one peer"); assert_eq!( @@ -84,9 +85,11 @@ async fn test_network_contacts_format() -> Result<(), Box addrs: vec![], network_contacts_url: Some(format!("{}/peers", mock_server.uri()).parse()?), local: false, + testnet: false, }; - let store = BootstrapCacheStore::from_args(args, config).await?; + let mut store = BootstrapCacheStore::empty(config)?; + store.initialize_from_peers_arg(&args).await?; let adddrs = store.get_addrs().collect::>(); assert_eq!( adddrs.len(), @@ -106,58 +109,6 @@ async fn test_network_contacts_format() -> Result<(), Box Ok(()) } -#[tokio::test] -async fn test_invalid_address_handling() -> Result<(), Box> { - let _guard = LogBuilder::init_single_threaded_tokio_test("address_format_tests", false); - - // Test various invalid address formats - let invalid_addrs = vec![ - "not-a-multiaddr", - "127.0.0.1", // IP only - "127.0.0.1:8080:extra", // Invalid socket addr - "/ip4/127.0.0.1", // Incomplete multiaddr - ]; - - for addr_str in invalid_addrs { - let (_temp_dir, config) = setup().await; // Fresh config for each test case - let args = PeersArgs { - first: false, - addrs: vec![], - network_contacts_url: None, - local: true, // Use local mode to avoid fetching from default endpoints - }; - - let store = BootstrapCacheStore::from_args(args.clone(), config.clone()).await?; - let addrs = store.get_addrs().collect::>(); - assert_eq!( - addrs.len(), - 0, - "Should have no peers from invalid address in env var: {}", - addr_str - ); - - // Also test direct args path - if let Ok(addr) = addr_str.parse::() { - let args_with_peer = PeersArgs { - first: false, - addrs: vec![addr], - network_contacts_url: None, - local: false, - }; - let store = BootstrapCacheStore::from_args(args_with_peer, config).await?; - let addrs = store.get_addrs().collect::>(); - assert_eq!( - addrs.len(), - 0, - "Should have no peers from invalid address in args: {}", - addr_str - ); - } - } - - Ok(()) -} - #[tokio::test] async fn test_socket_addr_format() -> Result<(), Box> { let _guard = LogBuilder::init_single_threaded_tokio_test("address_format_tests", false); @@ -170,13 +121,13 @@ async fn test_socket_addr_format() -> Result<(), Box> { addrs: vec![], network_contacts_url: None, local: true, // Use local mode to avoid getting peers from default endpoints + testnet: false, }; - let config = BootstrapConfig::empty() - .unwrap() - .with_cache_path(&cache_path); + let config = BootstrapCacheConfig::empty().with_cache_path(&cache_path); - let store = BootstrapCacheStore::from_args(args, config).await?; + let mut store = BootstrapCacheStore::empty(config)?; + store.initialize_from_peers_arg(&args).await?; let addrs = store.get_addrs().collect::>(); assert!(addrs.is_empty(), "Should have no peers in local mode"); @@ -195,13 +146,13 @@ async fn test_multiaddr_format() -> Result<(), Box> { addrs: vec![], network_contacts_url: None, local: true, // Use local mode to avoid getting peers from default endpoints + testnet: false, }; - let config = BootstrapConfig::empty() - .unwrap() - .with_cache_path(&cache_path); + let config = BootstrapCacheConfig::empty().with_cache_path(&cache_path); - let store = BootstrapCacheStore::from_args(args, config).await?; + let mut store = BootstrapCacheStore::empty(config)?; + store.initialize_from_peers_arg(&args).await?; let addrs = store.get_addrs().collect::>(); assert!(addrs.is_empty(), "Should have no peers in local mode"); @@ -220,13 +171,13 @@ async fn test_invalid_addr_format() -> Result<(), Box> { addrs: vec![], network_contacts_url: None, local: true, // Use local mode to avoid getting peers from default endpoints + testnet: false, }; - let config = BootstrapConfig::empty() - .unwrap() - .with_cache_path(&cache_path); + let config = BootstrapCacheConfig::empty().with_cache_path(&cache_path); - let store = BootstrapCacheStore::from_args(args, config).await?; + let mut store = BootstrapCacheStore::empty(config)?; + store.initialize_from_peers_arg(&args).await?; let addrs = store.get_addrs().collect::>(); assert!(addrs.is_empty(), "Should have no peers in local mode"); @@ -245,13 +196,13 @@ async fn test_mixed_addr_formats() -> Result<(), Box> { addrs: vec![], network_contacts_url: None, local: true, // Use local mode to avoid getting peers from default endpoints + testnet: false, }; - let config = BootstrapConfig::empty() - .unwrap() - .with_cache_path(&cache_path); + let config = BootstrapCacheConfig::empty().with_cache_path(&cache_path); - let store = BootstrapCacheStore::from_args(args, config).await?; + let mut store = BootstrapCacheStore::empty(config)?; + store.initialize_from_peers_arg(&args).await?; let addrs = store.get_addrs().collect::>(); assert!(addrs.is_empty(), "Should have no peers in local mode"); @@ -270,13 +221,13 @@ async fn test_socket_addr_conversion() -> Result<(), Box> addrs: vec![], network_contacts_url: None, local: true, // Use local mode to avoid getting peers from default endpoints + testnet: false, }; - let config = BootstrapConfig::empty() - .unwrap() - .with_cache_path(&cache_path); + let config = BootstrapCacheConfig::empty().with_cache_path(&cache_path); - let store = BootstrapCacheStore::from_args(args, config).await?; + let mut store = BootstrapCacheStore::empty(config)?; + store.initialize_from_peers_arg(&args).await?; let addrs = store.get_addrs().collect::>(); assert!(addrs.is_empty(), "Should have no peers in local mode"); @@ -295,13 +246,13 @@ async fn test_invalid_socket_addr() -> Result<(), Box> { addrs: vec![], network_contacts_url: None, local: true, // Use local mode to avoid getting peers from default endpoints + testnet: false, }; - let config = BootstrapConfig::empty() - .unwrap() - .with_cache_path(&cache_path); + let config = BootstrapCacheConfig::empty().with_cache_path(&cache_path); - let store = BootstrapCacheStore::from_args(args, config).await?; + let mut store = BootstrapCacheStore::empty(config)?; + store.initialize_from_peers_arg(&args).await?; let addrs = store.get_addrs().collect::>(); assert!(addrs.is_empty(), "Should have no peers in local mode"); @@ -320,13 +271,13 @@ async fn test_invalid_multiaddr() -> Result<(), Box> { addrs: vec![], network_contacts_url: None, local: true, // Use local mode to avoid getting peers from default endpoints + testnet: false, }; - let config = BootstrapConfig::empty() - .unwrap() - .with_cache_path(&cache_path); + let config = BootstrapCacheConfig::empty().with_cache_path(&cache_path); - let store = BootstrapCacheStore::from_args(args, config).await?; + let mut store = BootstrapCacheStore::empty(config)?; + store.initialize_from_peers_arg(&args).await?; let addrs = store.get_addrs().collect::>(); assert!(addrs.is_empty(), "Should have no peers in local mode"); @@ -345,13 +296,13 @@ async fn test_mixed_valid_invalid_addrs() -> Result<(), Box>(); assert!(addrs.is_empty(), "Should have no peers in local mode"); diff --git a/ant-bootstrap-cache/tests/cache_tests.rs b/ant-bootstrap/tests/cache_tests.rs similarity index 85% rename from ant-bootstrap-cache/tests/cache_tests.rs rename to ant-bootstrap/tests/cache_tests.rs index d3673c3206..aac95579a0 100644 --- a/ant-bootstrap-cache/tests/cache_tests.rs +++ b/ant-bootstrap/tests/cache_tests.rs @@ -6,7 +6,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use ant_bootstrap_cache::{BootstrapCacheStore, BootstrapConfig}; +use ant_bootstrap::{BootstrapCacheConfig, BootstrapCacheStore}; use ant_logging::LogBuilder; use libp2p::Multiaddr; use std::time::Duration; @@ -21,11 +21,9 @@ async fn test_cache_store_operations() -> Result<(), Box> let cache_path = temp_dir.path().join("cache.json"); // Create cache store with config - let config = BootstrapConfig::empty() - .unwrap() - .with_cache_path(&cache_path); + let config = BootstrapCacheConfig::empty().with_cache_path(&cache_path); - let mut cache_store = BootstrapCacheStore::new(config).await?; + let mut cache_store = BootstrapCacheStore::empty(config)?; // Test adding and retrieving peers let addr: Multiaddr = @@ -51,11 +49,9 @@ async fn test_cache_persistence() -> Result<(), Box> { let cache_path = temp_dir.path().join("cache.json"); // Create first cache store - let config = BootstrapConfig::empty() - .unwrap() - .with_cache_path(&cache_path); + let config = BootstrapCacheConfig::empty().with_cache_path(&cache_path); - let mut cache_store1 = BootstrapCacheStore::new(config.clone()).await?; + let mut cache_store1 = BootstrapCacheStore::empty(config.clone())?; // Add a peer and mark it as reliable let addr: Multiaddr = @@ -66,7 +62,8 @@ async fn test_cache_persistence() -> Result<(), Box> { cache_store1.sync_and_save_to_disk(true).await.unwrap(); // Create a new cache store with the same path - let cache_store2 = BootstrapCacheStore::new(config).await?; + let mut cache_store2 = BootstrapCacheStore::empty(config)?; + cache_store2.initialize_from_local_cache().await.unwrap(); let addrs = cache_store2.get_reliable_addrs().collect::>(); assert!(!addrs.is_empty(), "Cache should persist across instances"); @@ -84,10 +81,8 @@ async fn test_cache_reliability_tracking() -> Result<(), Box Result<(), Box> { let cache_path = temp_dir.path().join("cache.json"); // Create cache with small max_peers limit - let mut config = BootstrapConfig::empty() - .unwrap() - .with_cache_path(&cache_path); + let mut config = BootstrapCacheConfig::empty().with_cache_path(&cache_path); config.max_peers = 2; - let mut cache_store = BootstrapCacheStore::new(config).await?; + let mut cache_store = BootstrapCacheStore::empty(config)?; // Add three peers with distinct timestamps let mut addresses = Vec::new(); @@ -171,11 +164,9 @@ async fn test_cache_file_corruption() -> Result<(), Box> let cache_path = temp_dir.path().join("cache.json"); // Create cache with some peers - let config = BootstrapConfig::empty() - .unwrap() - .with_cache_path(&cache_path); + let config = BootstrapCacheConfig::empty().with_cache_path(&cache_path); - let mut cache_store = BootstrapCacheStore::new_without_init(config.clone()).await?; + let mut cache_store = BootstrapCacheStore::empty(config.clone())?; // Add a peer let addr: Multiaddr = @@ -189,7 +180,7 @@ async fn test_cache_file_corruption() -> Result<(), Box> tokio::fs::write(&cache_path, "invalid json content").await?; // Create a new cache store - it should handle the corruption gracefully - let mut new_cache_store = BootstrapCacheStore::new_without_init(config).await?; + let mut new_cache_store = BootstrapCacheStore::empty(config)?; let addrs = new_cache_store.get_addrs().collect::>(); assert!(addrs.is_empty(), "Cache should be empty after corruption"); diff --git a/ant-bootstrap-cache/tests/cli_integration_tests.rs b/ant-bootstrap/tests/cli_integration_tests.rs similarity index 82% rename from ant-bootstrap-cache/tests/cli_integration_tests.rs rename to ant-bootstrap/tests/cli_integration_tests.rs index ebc0bb86ea..f6c1c55925 100644 --- a/ant-bootstrap-cache/tests/cli_integration_tests.rs +++ b/ant-bootstrap/tests/cli_integration_tests.rs @@ -6,7 +6,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use ant_bootstrap_cache::{BootstrapCacheStore, BootstrapConfig, PeersArgs}; +use ant_bootstrap::{BootstrapCacheConfig, BootstrapCacheStore, PeersArgs}; use ant_logging::LogBuilder; use libp2p::Multiaddr; use std::env; @@ -17,12 +17,10 @@ use wiremock::{ Mock, MockServer, ResponseTemplate, }; -async fn setup() -> (TempDir, BootstrapConfig) { +async fn setup() -> (TempDir, BootstrapCacheConfig) { let temp_dir = TempDir::new().unwrap(); let cache_path = temp_dir.path().join("cache.json"); - let config = BootstrapConfig::empty() - .unwrap() - .with_cache_path(&cache_path); + let config = BootstrapCacheConfig::empty().with_cache_path(&cache_path); (temp_dir, config) } @@ -37,9 +35,11 @@ async fn test_first_flag() -> Result<(), Box> { addrs: vec![], network_contacts_url: None, local: false, + testnet: false, }; - let store = BootstrapCacheStore::from_args(args, config).await?; + let mut store = BootstrapCacheStore::empty(config.clone())?; + store.initialize_from_peers_arg(&args).await?; let addrs = store.get_addrs().collect::>(); assert!(addrs.is_empty(), "First node should have no addrs"); @@ -60,9 +60,11 @@ async fn test_peer_argument() -> Result<(), Box> { addrs: vec![peer_addr.clone()], network_contacts_url: None, local: false, + testnet: false, }; - let store = BootstrapCacheStore::from_args(args, config).await?; + let mut store = BootstrapCacheStore::empty(config.clone())?; + store.initialize_from_peers_arg(&args).await?; let addrs = store.get_addrs().collect::>(); assert_eq!(addrs.len(), 1, "Should have one addr"); assert_eq!(addrs[0].addr, peer_addr, "Should have the correct address"); @@ -87,13 +89,13 @@ async fn test_safe_peers_env() -> Result<(), Box> { addrs: vec![], network_contacts_url: None, local: false, + testnet: false, }; - let config = BootstrapConfig::empty() - .unwrap() - .with_cache_path(&cache_path); + let config = BootstrapCacheConfig::empty().with_cache_path(&cache_path); - let store = BootstrapCacheStore::from_args(args, config).await?; + let mut store = BootstrapCacheStore::empty(config.clone())?; + store.initialize_from_peers_arg(&args).await?; let addrs = store.get_addrs().collect::>(); // We should have multiple peers (env var + cache/endpoints) @@ -131,9 +133,11 @@ async fn test_network_contacts_fallback() -> Result<(), Box>(); assert_eq!( addrs.len(), @@ -152,9 +156,7 @@ async fn test_local_mode() -> Result<(), Box> { let cache_path = temp_dir.path().join("cache.json"); // Create a config with some peers in the cache - let config = BootstrapConfig::empty() - .unwrap() - .with_cache_path(&cache_path); + let config = BootstrapCacheConfig::empty().with_cache_path(&cache_path); // Create args with local mode enabled let args = PeersArgs { @@ -162,9 +164,11 @@ async fn test_local_mode() -> Result<(), Box> { addrs: vec![], network_contacts_url: None, local: true, + testnet: false, }; - let store = BootstrapCacheStore::from_args(args, config).await?; + let mut store = BootstrapCacheStore::empty(config.clone())?; + store.initialize_from_peers_arg(&args).await?; let addrs = store.get_addrs().collect::>(); assert!(addrs.is_empty(), "Local mode should have no peers"); @@ -188,18 +192,18 @@ async fn test_test_network_peers() -> Result<(), Box> { "/ip4/127.0.0.1/udp/8080/quic-v1/p2p/12D3KooWRBhwfeP2Y4TCx1SM6s9rUoHhR5STiGwxBhgFRcw3UERE" .parse()?; - let config = BootstrapConfig::empty() - .unwrap() - .with_cache_path(&cache_path); + let config = BootstrapCacheConfig::empty().with_cache_path(&cache_path); let args = PeersArgs { first: false, addrs: vec![peer_addr.clone()], network_contacts_url: None, local: false, + testnet: false, }; - let store = BootstrapCacheStore::from_args(args, config).await?; + let mut store = BootstrapCacheStore::empty(config.clone())?; + store.initialize_from_peers_arg(&args).await?; let addrs = store.get_addrs().collect::>(); assert_eq!(addrs.len(), 1, "Should have exactly one test network peer"); assert_eq!( @@ -228,9 +232,7 @@ async fn test_peers_update_cache() -> Result<(), Box> { "/ip4/127.0.0.1/udp/8080/quic-v1/p2p/12D3KooWRBhwfeP2Y4TCx1SM6s9rUoHhR5STiGwxBhgFRcw3UERE" .parse()?; - let config = BootstrapConfig::empty() - .unwrap() - .with_cache_path(&cache_path); + let config = BootstrapCacheConfig::empty().with_cache_path(&cache_path); // Create args with peers but no test network mode let args = PeersArgs { @@ -238,9 +240,11 @@ async fn test_peers_update_cache() -> Result<(), Box> { addrs: vec![peer_addr.clone()], network_contacts_url: None, local: false, + testnet: false, }; - let store = BootstrapCacheStore::from_args(args, config).await?; + let mut store = BootstrapCacheStore::empty(config.clone())?; + store.initialize_from_peers_arg(&args).await?; let addrs = store.get_addrs().collect::>(); assert_eq!(addrs.len(), 1, "Should have one peer"); assert_eq!(addrs[0].addr, peer_addr, "Should have the correct peer"); diff --git a/ant-bootstrap-cache/tests/integration_tests.rs b/ant-bootstrap/tests/integration_tests.rs similarity index 99% rename from ant-bootstrap-cache/tests/integration_tests.rs rename to ant-bootstrap/tests/integration_tests.rs index 53456c2af2..2bca68fabd 100644 --- a/ant-bootstrap-cache/tests/integration_tests.rs +++ b/ant-bootstrap/tests/integration_tests.rs @@ -6,7 +6,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use ant_bootstrap_cache::{BootstrapEndpoints, InitialPeerDiscovery}; +use ant_bootstrap::{BootstrapEndpoints, InitialPeerDiscovery}; use libp2p::Multiaddr; use tracing_subscriber::{fmt, EnvFilter}; use url::Url; diff --git a/ant-logging/src/layers.rs b/ant-logging/src/layers.rs index 2d26be3521..be0ac5668c 100644 --- a/ant-logging/src/layers.rs +++ b/ant-logging/src/layers.rs @@ -274,7 +274,7 @@ fn get_logging_targets(logging_env_value: &str) -> Result> ("antctl".to_string(), Level::TRACE), ("antctld".to_string(), Level::TRACE), // libs - ("ant_bootstrap_cache".to_string(), Level::TRACE), + ("ant_bootstrap".to_string(), Level::TRACE), ("ant_build_info".to_string(), Level::TRACE), ("ant_evm".to_string(), Level::TRACE), ("ant_logging".to_string(), Level::TRACE),