feat: network density estimation

maidsafe · Nov 22, 2024 · 4b0e20d · 4b0e20d
1 parent 77227ed
commit 4b0e20d
Show file tree

Hide file tree

Showing 5 changed files with 54 additions and 10 deletions.
diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml
@@ -1395,6 +1395,10 @@ jobs:
             exit 1
           fi
 
+      # Sleep for a while to allow network density sampling
+      - name: Sleep a while
+        run: sleep 300
+
       - name: Stop the local network and upload logs
         if: always()
         uses: maidsafe/sn-local-testnet-action@main

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/sn_networking/Cargo.toml b/sn_networking/Cargo.toml
@@ -37,6 +37,7 @@ libp2p = { version = "0.54.1", features = [
     "yamux",
     "websocket",
 ] }
+alloy = { version = "0.5.3", default-features = false, features = ["std", "reqwest-rustls-tls", "provider-anvil-node", "sol-types", "json", "signers", "contract", "signer-local", "network"] }
 async-trait = "0.1"
 bytes = { version = "1.0.1", features = ["serde"] }
 exponential-backoff = "2.0.0"

diff --git a/sn_networking/src/driver.rs b/sn_networking/src/driver.rs
@@ -29,6 +29,7 @@ use crate::{
     metrics::service::run_metrics_server, metrics::NetworkMetricsRecorder, MetricsRegistries,
 };
 use crate::{transport, NodeIssue};
+use alloy::primitives::U256;
 use futures::future::Either;
 use futures::StreamExt;
 #[cfg(feature = "local")]
@@ -923,13 +924,29 @@ impl SwarmDriver {
                         let closest_k_peers = self.get_closest_k_value_local_peers();
 
                         if let Some(distance) = self.get_responsbile_range_estimate(&closest_k_peers) {
-                            let network_density = self.network_density_samples.get_median();
-                            let ilog2 = if let Some(distance) = network_density {
+                            let (
+                                _index,
+                                _total_peers,
+                                peers_in_non_full_buckets,
+                                num_of_full_buckets,
+                                _kbucket_table_stats,
+                            ) = self.kbuckets_status();
+                            let estimated_network_size =
+                                Self::estimate_network_size(peers_in_non_full_buckets, num_of_full_buckets);
+                            // The entire Distance space is U256
+                            // The density can be estimated as: distance/space
+                            let density = U256::MAX / U256::from(estimated_network_size);
+                            let estimated_distance = density * U256::from(CLOSE_GROUP_SIZE);
+
+                            let sampled_distance = self.network_density_samples.get_median();
+                            let ilog2 = if let Some(distance) = sampled_distance {
                                 distance.ilog2()
                             } else {
                                 None
                             };
-                            info!("Set responsible range to {distance}, current sampled network density is {ilog2:?}({network_density:?})");
+                            info!("Set responsible range to {distance}, current sampled_distance is {ilog2:?}({sampled_distance:?}), \
+                                estimated_distance is {:?}({estimated_distance:?}) with network_size of {estimated_network_size}",
+                                estimated_distance.log2());
                             // set any new distance to farthest record in the store
                             self.swarm.behaviour_mut().kademlia.store_mut().set_distance_range(distance);
                             // the distance range within the replication_fetcher shall be in sync as well

diff --git a/sn_networking/src/event/mod.rs b/sn_networking/src/event/mod.rs
@@ -36,6 +36,9 @@ use std::{
 };
 use tokio::sync::oneshot;
 
+// (total_buckets, total_peers, peers_in_non_full_buckets, num_of_full_buckets, kbucket_table_stats)
+type KBucketStatus = (usize, usize, usize, usize, Vec<(usize, usize, u32)>);
+
 /// NodeEvent enum
 #[derive(CustomDebug)]
 pub(super) enum NodeEvent {
@@ -295,12 +298,8 @@ impl SwarmDriver {
         }
     }
 
-    /// Logs the kbuckets also records the bucket info.
-    pub(crate) fn log_kbuckets(&mut self, peer: &PeerId) {
-        let distance = NetworkAddress::from_peer(self.self_peer_id)
-            .distance(&NetworkAddress::from_peer(*peer));
-        info!("Peer {peer:?} has a {:?} distance to us", distance.ilog2());
-
+    /// Collect kbuckets status
+    pub(crate) fn kbuckets_status(&mut self) -> KBucketStatus {
         let mut kbucket_table_stats = vec![];
         let mut index = 0;
         let mut total_peers = 0;
@@ -327,6 +326,28 @@ impl SwarmDriver {
             }
             index += 1;
         }
+        (
+            index,
+            total_peers,
+            peers_in_non_full_buckets,
+            num_of_full_buckets,
+            kbucket_table_stats,
+        )
+    }
+
+    /// Logs the kbuckets also records the bucket info.
+    pub(crate) fn log_kbuckets(&mut self, peer: &PeerId) {
+        let distance = NetworkAddress::from_peer(self.self_peer_id)
+            .distance(&NetworkAddress::from_peer(*peer));
+        info!("Peer {peer:?} has a {:?} distance to us", distance.ilog2());
+
+        let (
+            index,
+            total_peers,
+            peers_in_non_full_buckets,
+            num_of_full_buckets,
+            kbucket_table_stats,
+        ) = self.kbuckets_status();
 
         let estimated_network_size =
             Self::estimate_network_size(peers_in_non_full_buckets, num_of_full_buckets);
@@ -353,7 +374,7 @@ impl SwarmDriver {
     }
 
     /// Estimate the number of nodes in the network
-    fn estimate_network_size(
+    pub(crate) fn estimate_network_size(
         peers_in_non_full_buckets: usize,
         num_of_full_buckets: usize,
     ) -> usize {