diff --git a/.env.example b/.env.example index 6ade5e55..cf06a8d3 100644 --- a/.env.example +++ b/.env.example @@ -3,6 +3,8 @@ RPC_URL_ETHEREUM_SEPOLIA=https://goerli.infura.io/v3/your-infura-api-key # this value is optional RPC_CHUNK_SIZE_ETHEREUM_SEPOLIA=2000 +RPC_URL_STARKNET_SEPOLIA=# if it's starknet make sure to use pathfinder + # Optional DRY_RUN_CAIRO_PATH= # path for dry run cairo SOUND_RUN_CAIRO_PATH= # path for sound run cairo diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d247b59d..7575e505 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,6 @@ jobs: run: | rustup component add clippy rustup component add rustfmt - - name: Install cargo-make - run: cargo install --debug cargo-make + - uses: taiki-e/install-action@just - name: Run clippy and formatter checks - run: cargo make run-ci-flow + run: just run-ci-flow diff --git a/Cargo.lock b/Cargo.lock index 30a118f9..09c984d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2182,6 +2182,7 @@ dependencies = [ "serde_with 2.3.3", "starknet", "starknet-crypto", + "starknet-types-core", "tempfile", "thiserror", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 05890d1e..1bf76b87 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,8 @@ [workspace] -resolver = "2" members = ["cli", "examples/private-input-module", "hdp"] [workspace.package] +resolver = "2" version = "0.4.0" edition = "2021" license-file = "LICENSE" @@ -37,6 +37,7 @@ rand = "0.8.4" regex = "1" starknet = "0.10.0" starknet-crypto = "0.6.1" +starknet-types-core = "0.1.0" cairo-lang-starknet-classes = "2.7.0" cairo-vm = "1.0.0-rc6" futures = "0.3.30" diff --git a/Makefile.toml b/Makefile.toml deleted file mode 100644 index 408c717c..00000000 --- a/Makefile.toml +++ /dev/null @@ -1,29 +0,0 @@ -[env] -CARGO_MAKE_EXTEND_WORKSPACE_MAKEFILE = true - -[tasks.format] -install_crate = "rustfmt" -command = "cargo" -args = ["fmt", "--", "--check"] -description = "Run rustfmt to check the code formatting without making changes." - -[tasks.clean] -command = "cargo" -args = ["clean"] -description = "Clean up the project by removing the target directory." - -[tasks.clippy] -command = "cargo" -args = ["clippy", "--all-targets", "--all-features", "--", "-Dwarnings"] -description = "Run clippy to catch common mistakes and improve your Rust code." - -[tasks.test] -workspace = false -command = "cargo" -args = ["llvm-cov", "nextest", "--features", "test_utils"] -description = "Execute all unit tests in the workspace." - -[tasks.run-ci-flow] -workspace = false -description = "Run the entire CI pipeline including format, clippy, and test checks." -dependencies = ["format", "clippy", "test"] diff --git a/hdp/Cargo.toml b/hdp/Cargo.toml index 6ff37feb..a4baafff 100644 --- a/hdp/Cargo.toml +++ b/hdp/Cargo.toml @@ -24,6 +24,7 @@ serde = { workspace = true } serde_with = { workspace = true } serde_json = { workspace = true } starknet-crypto = { workspace = true } +starknet-types-core = { workspace = true } starknet = { workspace = true } thiserror.workspace = true alloy-merkle-tree = { workspace = true } diff --git a/hdp/src/provider/error.rs b/hdp/src/provider/error.rs index c82699fb..23678940 100644 --- a/hdp/src/provider/error.rs +++ b/hdp/src/provider/error.rs @@ -1,9 +1,8 @@ +use alloy::primitives::BlockNumber; use thiserror::Error; use crate::provider::indexer::IndexerError; -use super::evm::rpc::RpcProviderError; - /// Error type for provider #[derive(Error, Debug)] pub enum ProviderError { @@ -34,3 +33,22 @@ pub enum ProviderError { #[error("Fetch key error: {0}")] FetchKeyError(String), } + +/// Error from [`RpcProvider`] +#[derive(Error, Debug)] +pub enum RpcProviderError { + #[error("Failed to send proofs with mpsc")] + MpscError( + #[from] + tokio::sync::mpsc::error::SendError<( + BlockNumber, + alloy::rpc::types::EIP1186AccountProofResponse, + )>, + ), + + #[error("Failed to fetch proofs: {0}")] + ReqwestError(#[from] reqwest::Error), + + #[error("Failed to parse response: {0}")] + SerdeJsonError(#[from] serde_json::Error), +} diff --git a/hdp/src/provider/evm/rpc.rs b/hdp/src/provider/evm/rpc.rs index 24d33e7e..e315b00f 100644 --- a/hdp/src/provider/evm/rpc.rs +++ b/hdp/src/provider/evm/rpc.rs @@ -15,25 +15,13 @@ use alloy::{ }; use futures::future::join_all; use reqwest::Url; -use thiserror::Error; use tokio::sync::{ mpsc::{self, Sender}, RwLock, }; use tracing::debug; -/// Error from [`RpcProvider`] -#[derive(Error, Debug)] -pub enum RpcProviderError { - #[error("Failed to send proofs with mpsc")] - MpscError( - #[from] - tokio::sync::mpsc::error::SendError<( - BlockNumber, - alloy::rpc::types::EIP1186AccountProofResponse, - )>, - ), -} +use crate::provider::error::RpcProviderError; /// RPC provider for fetching data from Ethereum RPC /// It is a wrapper around the alloy provider, using eth_getProof for fetching account and storage proofs diff --git a/hdp/src/provider/starknet/mod.rs b/hdp/src/provider/starknet/mod.rs index a8c241cb..93a521a5 100644 --- a/hdp/src/provider/starknet/mod.rs +++ b/hdp/src/provider/starknet/mod.rs @@ -1 +1,3 @@ -pub struct StarknetProvider {} +pub mod provider; +pub mod rpc; +pub mod types; diff --git a/hdp/src/provider/starknet/provider.rs b/hdp/src/provider/starknet/provider.rs new file mode 100644 index 00000000..7b03d600 --- /dev/null +++ b/hdp/src/provider/starknet/provider.rs @@ -0,0 +1,121 @@ +use std::{collections::HashMap, time::Instant}; + +use alloy::primitives::BlockNumber; +use itertools::Itertools; +use starknet_types_core::felt::Felt; +use tracing::info; + +use crate::provider::{config::ProviderConfig, error::ProviderError, indexer::Indexer}; + +use super::{rpc::RpcProvider, types::GetProofOutput}; + +type AccountProofsResult = Result, ProviderError>; +type StorageProofsResult = Result, ProviderError>; + +pub struct StarknetProvider { + /// Account and storage trie provider + pub(crate) rpc_provider: RpcProvider, + /// Header provider + //TODO: indexer is not supported for starknet yet + pub(crate) _header_provider: Indexer, +} + +#[cfg(feature = "test_utils")] +impl Default for StarknetProvider { + fn default() -> Self { + Self::new(&ProviderConfig::default()) + } +} + +impl StarknetProvider { + pub fn new(config: &ProviderConfig) -> Self { + let rpc_provider = RpcProvider::new(config.rpc_url.to_owned(), config.max_requests); + let indexer = Indexer::new(config.chain_id); + Self { + rpc_provider, + _header_provider: indexer, + } + } + + /// Fetches the account proofs for the given block range. + /// The account proofs are fetched from the RPC provider. + /// + /// Return: + /// - Account proofs mapped by block number + pub async fn get_range_of_account_proofs( + &self, + from_block: BlockNumber, + to_block: BlockNumber, + increment: u64, + address: Felt, + ) -> AccountProofsResult { + let start_fetch = Instant::now(); + + let target_blocks_batch: Vec> = + self._chunk_block_range(from_block, to_block, increment); + + let mut fetched_accounts_proofs_with_blocks_map = HashMap::new(); + for target_blocks in target_blocks_batch { + fetched_accounts_proofs_with_blocks_map.extend( + self.rpc_provider + .get_account_proofs(target_blocks, address) + .await?, + ); + } + + let duration = start_fetch.elapsed(); + info!("time taken (Account Proofs Fetch): {:?}", duration); + + Ok(fetched_accounts_proofs_with_blocks_map) + } + + /// Fetches the storage proofs for the given block range. + /// The storage proofs are fetched from the RPC provider. + /// + /// Return: + /// - Storage proofs mapped by block number + pub async fn get_range_of_storage_proofs( + &self, + from_block: BlockNumber, + to_block: BlockNumber, + increment: u64, + address: Felt, + storage_slot: Felt, + ) -> StorageProofsResult { + let start_fetch = Instant::now(); + + let target_blocks_batch: Vec> = + self._chunk_block_range(from_block, to_block, increment); + + let mut processed_accounts = HashMap::new(); + for target_blocks in target_blocks_batch { + processed_accounts.extend( + self.rpc_provider + .get_storage_proofs(target_blocks, address, storage_slot) + .await?, + ); + } + + let duration = start_fetch.elapsed(); + info!("time taken (Storage Proofs Fetch): {:?}", duration); + + Ok(processed_accounts) + } + + /// Chunks the block range into smaller ranges of 800 blocks. + /// This is to avoid fetching too many blocks at once from the RPC provider. + /// This is meant to use with data lake definition, which have sequential block numbers + pub(crate) fn _chunk_block_range( + &self, + from_block: BlockNumber, + to_block: BlockNumber, + increment: u64, + ) -> Vec> { + (from_block..=to_block) + .step_by(increment as usize) + .chunks(800) + .into_iter() + .map(|chunk| chunk.collect()) + .collect() + } +} diff --git a/hdp/src/provider/starknet/rpc.rs b/hdp/src/provider/starknet/rpc.rs new file mode 100644 index 00000000..542f65ac --- /dev/null +++ b/hdp/src/provider/starknet/rpc.rs @@ -0,0 +1,276 @@ +use alloy::primitives::BlockNumber; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, + time::Instant, +}; + +use futures::future::join_all; +use reqwest::{Client, Url}; +use serde_json::json; +use starknet_types_core::felt::Felt; +use tokio::sync::{ + mpsc::{self, Sender}, + RwLock, +}; +use tracing::{debug, error}; + +use crate::provider::error::RpcProviderError; + +use super::types::GetProofOutput; + +/// !Note: have to use pathfinder node as we need `pathfinder_getProof` +pub struct RpcProvider { + client: reqwest::Client, + url: Url, + chunk_size: u64, +} + +impl RpcProvider { + pub fn new(rpc_url: Url, chunk_size: u64) -> Self { + Self { + client: Client::new(), + url: rpc_url, + chunk_size, + } + } + + /// Get account with proof in given vector of blocks + pub async fn get_account_proofs( + &self, + blocks: Vec, + address: Felt, + ) -> Result, RpcProviderError> { + self.get_proofs(blocks, address, None).await + } + + /// Get storage with proof in given vector of blocks and slot + pub async fn get_storage_proofs( + &self, + block_range: Vec, + address: Felt, + storage_key: Felt, + ) -> Result, RpcProviderError> { + self.get_proofs(block_range, address, Some(storage_key)) + .await + } + + async fn get_proofs( + &self, + blocks: Vec, + address: Felt, + storage_key: Option, + ) -> Result, RpcProviderError> { + let start_fetch = Instant::now(); + let (rpc_sender, mut rx) = mpsc::channel::<(BlockNumber, GetProofOutput)>(32); + self.spawn_proof_fetcher(rpc_sender, blocks, address, storage_key); + + let mut fetched_proofs = HashMap::new(); + while let Some((block_number, proof)) = rx.recv().await { + fetched_proofs.insert(block_number, proof); + } + let duration = start_fetch.elapsed(); + debug!("time taken (Fetch): {:?}", duration); + + Ok(fetched_proofs) + } + + fn spawn_proof_fetcher( + &self, + rpc_sender: Sender<(BlockNumber, GetProofOutput)>, + blocks: Vec, + address: Felt, + storage_key: Option, + ) { + let chunk_size = self.chunk_size; + let provider_clone = self.client.clone(); + let target_blocks_length = blocks.len(); + let url = self.url.clone(); + + debug!( + "fetching proofs for {}, with chunk size: {}", + address, chunk_size + ); + + tokio::spawn(async move { + let mut try_count = 0; + let blocks_map = Arc::new(RwLock::new(HashSet::::new())); + + while blocks_map.read().await.len() < target_blocks_length { + try_count += 1; + if try_count > 50 { + panic!("❗️❗️❗️ Too many retries, failed to fetch all blocks") + } + let fetched_blocks_clone = blocks_map.read().await.clone(); + + let blocks_to_fetch: Vec = blocks + .iter() + .filter(|block_number| !fetched_blocks_clone.contains(block_number)) + .take(chunk_size as usize) + .cloned() + .collect(); + + let fetch_futures = blocks_to_fetch + .into_iter() + .map(|block_number| { + let fetched_blocks_clone = blocks_map.clone(); + let rpc_sender = rpc_sender.clone(); + let provider_clone = provider_clone.clone(); + let url = url.clone(); + async move { + let proof = pathfinder_get_proof( + &provider_clone, + url, + address, + block_number, + storage_key, + ) + .await; + handle_proof_result( + proof, + block_number, + fetched_blocks_clone, + rpc_sender, + ) + .await; + } + }) + .collect::>(); + + join_all(fetch_futures).await; + } + }); + } +} + +/// Fetches proof (account or storage) for a given block number +async fn pathfinder_get_proof( + provider: &reqwest::Client, + url: Url, + address: Felt, + block_number: BlockNumber, + storage_key: Option, +) -> Result { + let mut keys = Vec::new(); + if let Some(key) = storage_key { + keys.push(key.to_hex_string()); + } + + let request = json!({ + "jsonrpc": "2.0", + "id": "0", + "method": "pathfinder_getProof", + "params": { + "block_id": {"block_number": block_number}, + "contract_address": format!("{}", address.to_hex_string()), + "keys": keys + } + }); + + let response = provider.post(url).json(&request).send().await?; + let response_json = + serde_json::from_str::(&response.text().await?)?["result"].clone(); + let get_proof_output: GetProofOutput = serde_json::from_value(response_json)?; + Ok(get_proof_output) +} + +async fn handle_proof_result( + proof: Result, + block_number: BlockNumber, + blocks_map: Arc>>, + rpc_sender: Sender<(BlockNumber, GetProofOutput)>, +) { + match proof { + Ok(proof) => { + blocks_map.write().await.insert(block_number); + rpc_sender.send((block_number, proof)).await.unwrap(); + } + Err(e) => { + error!("❗️❗️❗️ Error fetching proof: {:?}", e); + } + } +} +#[cfg(test)] +mod tests { + use core::str::FromStr; + + use super::*; + use reqwest::Url; + + const PATHFINDER_URL: &str = "https://pathfinder.sepolia.iosis.tech/"; + + fn test_provider() -> RpcProvider { + RpcProvider::new(Url::from_str(PATHFINDER_URL).unwrap(), 100) + } + + #[tokio::test] + async fn test_get_100_range_storage_with_proof() { + // TODO: why the storage proof returns same value as account proof + let target_block_start = 156600; + let target_block_end = 156700; + let target_block_range = (target_block_start..=target_block_end).collect::>(); + let provider = test_provider(); + let proof = provider + .get_storage_proofs( + target_block_range.clone(), + Felt::from_str("0x23371b227eaecd8e8920cd429d2cd0f3fee6abaacca08d3ab82a7cdd") + .unwrap(), + Felt::from_str("0x2").unwrap(), + ) + .await + .unwrap(); + + assert_eq!(proof.len(), target_block_range.len()); + let output = proof.get(&target_block_start).unwrap(); + println!("Proof: {:?}", output); + assert_eq!( + output.state_commitment.unwrap(), + Felt::from_str("0x26da0f5f0849cf69b4872ef5dced3ec68ce28c5e3f53207280113abb7feb158") + .unwrap() + ); + + assert_eq!(output.contract_proof.len(), 23); + + assert_eq!( + output.class_commitment.unwrap(), + Felt::from_str("0x46c1a0374b8ccf8d928e62ef40974304732c8a28f10b2c494adfabfcff0fa0a") + .unwrap() + ); + + assert!(output.contract_data.is_none()); + } + + #[tokio::test] + async fn test_get_100_range_account_with_proof() { + let target_block_start = 156600; + let target_block_end = 156700; + let target_block_range = (target_block_start..=target_block_end).collect::>(); + let provider = test_provider(); + let proof = provider + .get_account_proofs( + target_block_range.clone(), + Felt::from_str("0x23371b227eaecd8e8920cd429d2cd0f3fee6abaacca08d3ab82a7cdd") + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(proof.len(), target_block_range.len()); + let output = proof.get(&target_block_start).unwrap(); + println!("Proof: {:?}", output); + assert_eq!( + output.state_commitment.unwrap(), + Felt::from_str("0x26da0f5f0849cf69b4872ef5dced3ec68ce28c5e3f53207280113abb7feb158") + .unwrap() + ); + assert_eq!(output.contract_proof.len(), 23); + + assert_eq!( + output.class_commitment.unwrap(), + Felt::from_str("0x46c1a0374b8ccf8d928e62ef40974304732c8a28f10b2c494adfabfcff0fa0a") + .unwrap() + ); + + assert!(output.contract_data.is_none()); + } +} diff --git a/hdp/src/provider/starknet/types.rs b/hdp/src/provider/starknet/types.rs new file mode 100644 index 00000000..3ff53a26 --- /dev/null +++ b/hdp/src/provider/starknet/types.rs @@ -0,0 +1,82 @@ +use serde::{Deserialize, Serialize}; +use serde_with::skip_serializing_none; +use starknet_types_core::{felt::Felt, hash::StarkHash}; + +/// Codebase is from https://github.com/eqlabs/pathfinder/tree/ae81d84b7c4157891069bd02ef810a29b60a94e3 + +/// Holds the membership/non-membership of a contract and its associated +/// contract contract if the contract exists. +#[derive(Debug, Serialize, Deserialize)] +#[skip_serializing_none] +pub struct GetProofOutput { + /// The global state commitment for Starknet 0.11.0 blocks onwards, if + /// absent the hash of the first node in the + /// [contract_proof](GetProofOutput#contract_proof) is the global state + /// commitment. + pub state_commitment: Option, + /// Required to verify that the hash of the class commitment and the root of + /// the [contract_proof](GetProofOutput::contract_proof) matches the + /// [state_commitment](Self#state_commitment). Present only for Starknet + /// blocks 0.11.0 onwards. + pub class_commitment: Option, + + /// Membership / Non-membership proof for the queried contract + pub contract_proof: Vec, + + /// Additional contract data if it exists. + pub contract_data: Option, +} + +/// A node in a Starknet patricia-merkle trie. +/// +/// See pathfinders merkle-tree crate for more information. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum TrieNode { + #[serde(rename = "binary")] + Binary { left: Felt, right: Felt }, + #[serde(rename = "edge")] + Edge { child: Felt, path: Path }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Path { + len: u64, + value: String, +} + +impl TrieNode { + pub fn hash(&self) -> Felt { + match self { + TrieNode::Binary { left, right } => H::hash(left, right), + TrieNode::Edge { child, path } => { + let bytes: [u8; 32] = path.value.as_bytes().try_into().unwrap(); + let mut length = [0; 32]; + // Safe as len() is guaranteed to be <= 251 + length[31] = bytes.len() as u8; + + let length = Felt::from_bytes_be(&length); + let path = Felt::from_bytes_be(&bytes); + H::hash(child, &path) + length + } + } + } +} + +/// Holds the data and proofs for a specific contract. +#[derive(Debug, Serialize, Deserialize)] +pub struct ContractData { + /// Required to verify the contract state hash to contract root calculation. + class_hash: Felt, + /// Required to verify the contract state hash to contract root calculation. + nonce: Felt, + + /// Root of the Contract state tree + root: Felt, + + /// This is currently just a constant = 0, however it might change in the + /// future. + contract_state_hash_version: Felt, + + /// The proofs associated with the queried storage values + storage_proofs: Vec>, +} diff --git a/justfile b/justfile new file mode 100644 index 00000000..b698c87f --- /dev/null +++ b/justfile @@ -0,0 +1,22 @@ +# Set environment variable +export CARGO_MAKE_EXTEND_WORKSPACE_MAKEFILE := "true" + +# Run rustfmt to check the code formatting without making changes +format: + cargo fmt -- --check + +# Clean up the project by removing the target directory +clean: + cargo clean + +# Run clippy to catch common mistakes and improve your Rust code +clippy: + cargo clippy --all-targets --all-features -- -Dwarnings + +# Execute all unit tests in the workspace +test: + cargo llvm-cov nextest --features test_utils + +# Run the entire CI pipeline including format, clippy, and test checks +run-ci-flow: format clippy test + @echo "CI flow completed"