fetcher progress bars

Cargo.toml

@@ -39,6 +39,7 @@ strum_macros = "0.26.4"
 thiserror = "1.0.64"
 tiny-keccak = { version = "2.0.2", features = ["keccak"] }
 tokio = { version = "1.41.1", features = ["full"] }
+indicatif = "0.17.9"
 
 dry_hint_processor = { path = "crates/dry_hint_processor" }
 eth_essentials_cairo_vm_hints = { path = "packages/eth_essentials/cairo_vm_hints"}

crates/fetcher/Cargo.toml

@@ -11,6 +11,7 @@ futures.workspace = true
 serde_json.workspace = true
 thiserror.workspace = true
 tokio.workspace = true
+indicatif.workspace = true
 
 indexer.workspace = true
 types.workspace = true

crates/fetcher/src/main.rs

@@ -8,14 +8,15 @@ use clap::{Parser, ValueHint};
 use dry_hint_processor::syscall_handler::evm::{self, SyscallHandler};
 use futures::{FutureExt, StreamExt};
 use indexer::types::IndexerError;
+use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
 use proof_keys::ProofKeys;
 use std::{collections::HashSet, fs, num::ParseIntError, path::PathBuf};
 use thiserror::Error;
 use types::proofs::{account::Account, storage::Storage, HeaderMmrMeta, Proofs};
 
 pub mod proof_keys;
 
-const BUFFER_UNORDERED: usize = 10;
+const BUFFER_UNORDERED: usize = 50;
 
 #[derive(Parser, Debug)]
 #[clap(author, version, about, long_about = None)]
@@ -30,6 +31,11 @@ struct Args {
 async fn main() -> Result<(), FetcherError> {
     let args = Args::try_parse_from(std::env::args()).map_err(FetcherError::Args)?;
 
+    let multi_progress = MultiProgress::new();
+    let progress_style = ProgressStyle::with_template("[{elapsed_precise}] [{bar:40}] {pos}/{len} {msg}")
+        .unwrap()
+        .progress_chars("=> ");
+
     let input_file = fs::read(args.filename)?;
     let syscall_handler = serde_json::from_slice::<SyscallHandler>(&input_file)?;
 
@@ -48,13 +54,24 @@ async fn main() -> Result<(), FetcherError> {
         }
     }
 
+    let pb_header_keys = multi_progress.add(ProgressBar::new(proof_keys.header_keys.len() as u64));
+    let pb_account_keys = multi_progress.add(ProgressBar::new(proof_keys.account_keys.len() as u64));
+    let pb_storage_keys = multi_progress.add(ProgressBar::new(proof_keys.storage_keys.len() as u64));
+    pb_header_keys.set_style(progress_style.clone());
+    pb_header_keys.set_message("header_keys");
+    pb_account_keys.set_style(progress_style.clone());
+    pb_account_keys.set_message("account_keys");
+    pb_storage_keys.set_style(progress_style);
+    pb_storage_keys.set_message("storage_keys");
+
     let mut headers_with_mmr: HashSet<HeaderMmrMeta> = HashSet::default();
 
     let mut headers_with_mmr_fut = futures::stream::iter(proof_keys.header_keys.iter().map(ProofKeys::fetch_header_proof).map(|f| f.boxed_local()))
         .buffer_unordered(BUFFER_UNORDERED);
 
     while let Some(Ok(item)) = headers_with_mmr_fut.next().await {
         headers_with_mmr.insert(item);
+        pb_header_keys.inc(1);
     }
 
     let mut accounts: HashSet<Account> = HashSet::default();
@@ -71,6 +88,7 @@ async fn main() -> Result<(), FetcherError> {
     while let Some(Ok((header_with_mmr, account))) = accounts_fut.next().await {
         headers_with_mmr.insert(header_with_mmr);
         accounts.insert(account);
+        pb_account_keys.inc(1);
     }
 
     let mut storages: HashSet<Storage> = HashSet::default();
@@ -88,6 +106,7 @@ async fn main() -> Result<(), FetcherError> {
         headers_with_mmr.insert(header_with_mmr.clone());
         accounts.insert(account);
         storages.insert(storage);
+        pb_storage_keys.inc(1);
     }
 
     let proofs = Proofs {