proof fetcher

Cargo.toml

@@ -2,10 +2,11 @@
 resolver = "2"
 
 members = [
+    "crates/cairo_types",
+    "crates/dry_run",
     "crates/hdp_hint_processor",
+    "crates/fetcher",
     "crates/provider",
-    "crates/cairo_types",
-    "crates/dry_run"
 ]
 
 [workspace.dependencies]

README.md

@@ -23,7 +23,7 @@ Before running the program, prepare the input data. The inputs are provided via
 To run the program, use:
 
 ```bash
-make run-hdp
+cargo run --bin dry_run -- build/compiled_cairo_files/contract_dry_run.json --program_input rust_input.json --program_output a.json --layout all_cairo
 ```
 
 The program will output the results root and tasks root. These roots can be used to extract the results from the on-chain contract.

crates/fetcher/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "fetcher"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+hdp_hint_processor.workspace = true
+cairo-vm.workspace = true
+clap.workspace = true
+thiserror.workspace = true
+tokio.workspace = true
+serde_json.workspace = true
+alloy.workspace = true
+hex.workspace = true
+
+provider.workspace = true

crates/fetcher/src/lib.rs

@@ -0,0 +1,26 @@
+use alloy::hex::FromHexError;
+use provider::indexer::types::IndexerError;
+use std::num::ParseIntError;
+use thiserror::Error;
+
+pub mod proof_keys;
+
+#[derive(Error, Debug)]
+pub enum FetcherError {
+    #[error(transparent)]
+    Args(#[from] clap::error::Error),
+    #[error("Output Error: {0}")]
+    Output(String),
+    #[error(transparent)]
+    IO(#[from] std::io::Error),
+    #[error(transparent)]
+    Indexer(#[from] IndexerError),
+    #[error(transparent)]
+    ParseIntError(#[from] ParseIntError),
+    #[error(transparent)]
+    FromHexError(#[from] FromHexError),
+    #[error(transparent)]
+    SerdeJson(#[from] serde_json::Error),
+    #[error("Internal Error: {0}")]
+    InternalError(String),
+}

crates/fetcher/src/main.rs

@@ -0,0 +1,85 @@
+#![forbid(unsafe_code)]
+#![allow(async_fn_in_trait)]
+use clap::{Parser, ValueHint};
+use fetcher::{proof_keys::ProofKeys, FetcherError};
+use hdp_hint_processor::{
+    hint_processor::models::proofs::{account::Account, header::Header, mmr::MmrMeta, storage::Storage, Proofs},
+    syscall_handler::evm::{self, dryrun::SyscallHandler},
+};
+use std::{collections::HashSet, fs, path::PathBuf};
+
+pub mod proof_keys;
+
+#[derive(Parser, Debug)]
+#[clap(author, version, about, long_about = None)]
+struct Args {
+    #[clap(value_parser, value_hint=ValueHint::FilePath)]
+    filename: PathBuf,
+    #[structopt(long = "program_output")]
+    program_output: PathBuf,
+}
+
+fn main() -> Result<(), FetcherError> {
+    let args = Args::try_parse_from(std::env::args()).map_err(FetcherError::Args)?;
+
+    let input_file = fs::read(args.filename)?;
+    let syscall_handler = serde_json::from_slice::<SyscallHandler>(&input_file)?;
+
+    let mut proof_keys = ProofKeys::default();
+    for key in syscall_handler.call_contract_handler.key_set {
+        match key {
+            evm::dryrun::DryRunKey::Account(value) => {
+                proof_keys.account_keys.insert(value);
+            }
+            evm::dryrun::DryRunKey::Header(value) => {
+                proof_keys.header_keys.insert(value);
+            }
+            evm::dryrun::DryRunKey::Storage(value) => {
+                proof_keys.storage_keys.insert(value);
+            }
+        }
+    }
+
+    let mut headers_with_mmr = proof_keys
+        .header_keys
+        .iter()
+        .map(ProofKeys::fetch_header_proof)
+        .collect::<Result<HashSet<(MmrMeta, Header)>, FetcherError>>()?;
+
+    let mut accounts: HashSet<Account> = HashSet::default();
+
+    for (header_with_mmr, account) in proof_keys
+        .account_keys
+        .iter()
+        .map(ProofKeys::fetch_account_proof)
+        .collect::<Result<Vec<((MmrMeta, Header), Account)>, FetcherError>>()?
+        .into_iter()
+    {
+        headers_with_mmr.insert(header_with_mmr);
+        accounts.insert(account);
+    }
+
+    let mut storages: HashSet<Storage> = HashSet::default();
+
+    for (header_with_mmr, account, storage) in proof_keys
+        .storage_keys
+        .iter()
+        .map(ProofKeys::fetch_storage_proof)
+        .collect::<Result<Vec<((MmrMeta, Header), Account, Storage)>, FetcherError>>()?
+        .into_iter()
+    {
+        headers_with_mmr.insert(header_with_mmr);
+        accounts.insert(account);
+        storages.insert(storage);
+    }
+
+    let proofs = Proofs {
+        headers_with_mmr: headers_with_mmr.into_iter().collect(),
+        accounts: accounts.into_iter().collect(),
+        storages: storages.into_iter().collect(),
+        ..Default::default()
+    };
+    fs::write(args.program_output, serde_json::to_vec(&proofs).map_err(|e| FetcherError::IO(e.into()))?)?;
+
+    Ok(())
+}

crates/fetcher/src/proof_keys.rs

@@ -0,0 +1,125 @@
+use alloy::{
+    hex::FromHexError,
+    primitives::Bytes,
+    providers::{Provider, RootProvider},
+    transports::http::{reqwest::Url, Client, Http},
+};
+use hdp_hint_processor::{
+    hint_processor::models::proofs::{
+        account::Account,
+        header::{Header, HeaderProof},
+        mmr::MmrMeta,
+        mpt::MPTProof,
+        storage::Storage,
+    },
+    syscall_handler::keys,
+};
+use provider::{
+    indexer::{
+        types::{BlockHeader, IndexerQuery},
+        Indexer,
+    },
+    RPC,
+};
+use std::{collections::HashSet, env};
+
+use crate::FetcherError;
+
+#[derive(Debug, Default)]
+pub struct ProofKeys {
+    pub header_keys: HashSet<keys::header::Key>,
+    pub account_keys: HashSet<keys::account::Key>,
+    pub storage_keys: HashSet<keys::storage::Key>,
+}
+
+impl ProofKeys {
+    pub fn fetch_header_proof(key: &keys::header::Key) -> Result<(MmrMeta, Header), FetcherError> {
+        let provider = Indexer::default();
+        let runtime = tokio::runtime::Runtime::new().unwrap();
+
+        // Fetch proof response
+        let response = runtime.block_on(async {
+            provider
+                .get_headers_proof(IndexerQuery::new(key.chain_id, key.block_number, key.block_number))
+                .await
+        })?;
+
+        // Extract MMR metadata
+        let mmr_meta = MmrMeta {
+            id: u64::from_str_radix(&response.mmr_meta.mmr_id[2..], 16)?,
+            size: response.mmr_meta.mmr_size,
+            root: response.mmr_meta.mmr_root.parse()?,
+            chain_id: key.chain_id,
+            peaks: response
+                .mmr_meta
+                .mmr_peaks
+                .iter()
+                .map(|peak| peak.parse())
+                .collect::<Result<Vec<Bytes>, FromHexError>>()?,
+        };
+
+        // Retrieve MMR proof
+        let mmr_proof = response
+            .headers
+            .get(&key.block_number)
+            .ok_or_else(|| FetcherError::InternalError("block not found".into()))?;
+
+        // Parse RLP
+        let rlp = match &mmr_proof.block_header {
+            BlockHeader::RlpString(rlp) => rlp,
+            _ => return Err(FetcherError::InternalError("wrong rlp format".into())),
+        };
+
+        // Construct Header
+        let header = Header {
+            rlp: rlp.parse()?,
+            proof: HeaderProof {
+                leaf_idx: mmr_proof.element_index,
+                mmr_path: mmr_proof
+                    .siblings_hashes
+                    .iter()
+                    .map(|hash| hash.parse())
+                    .collect::<Result<Vec<Bytes>, FromHexError>>()?,
+            },
+        };
+
+        Ok((mmr_meta, header))
+    }
+
+    pub fn fetch_account_proof(key: &keys::account::Key) -> Result<((MmrMeta, Header), Account), FetcherError> {
+        let runtime = tokio::runtime::Runtime::new().unwrap();
+        let provider = RootProvider::<Http<Client>>::new_http(Url::parse(&env::var(RPC).unwrap()).unwrap());
+        let value = runtime
+            .block_on(async { provider.get_proof(key.address, vec![]).block_id(key.block_number.into()).await })
+            .map_err(|e| FetcherError::InternalError(e.to_string()))?;
+        Ok((
+            Self::fetch_header_proof(&key.to_owned().into())?,
+            Account::new(value.address, vec![MPTProof::new(key.block_number, value.account_proof)]),
+        ))
+    }
+
+    pub fn fetch_storage_proof(key: &keys::storage::Key) -> Result<((MmrMeta, Header), Account, Storage), FetcherError> {
+        let runtime = tokio::runtime::Runtime::new().unwrap();
+        let provider = RootProvider::<Http<Client>>::new_http(Url::parse(&env::var(RPC).unwrap()).unwrap());
+        let value = runtime
+            .block_on(async {
+                provider
+                    .get_proof(key.address, vec![key.storage_slot])
+                    .block_id(key.block_number.into())
+                    .await
+            })
+            .map_err(|e| FetcherError::InternalError(e.to_string()))?;
+        Ok((
+            Self::fetch_header_proof(&key.to_owned().into())?,
+            Account::new(value.address, vec![MPTProof::new(key.block_number, value.account_proof)]),
+            Storage::new(
+                value.address,
+                key.storage_slot,
+                vec![MPTProof::new(
+                    key.block_number,
+                    value.storage_proof.into_iter().flat_map(|f| f.proof).collect(),
+                )],
+            ),
+        ))
+    }
+}

crates/hdp_hint_processor/src/hint_processor/models/proofs/mmr.rs

@@ -2,7 +2,7 @@ use alloy::primitives::Bytes;
 use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
 
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Eq, Hash)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Eq, Hash, Default)]
 #[serde_as]
 pub struct MmrMeta {
     pub id: u64,

crates/hdp_hint_processor/src/hint_processor/models/proofs/mod.rs

@@ -14,10 +14,11 @@ use serde::{Deserialize, Serialize};
 use storage::Storage;
 use transaction::Transaction;
 
-#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Default)]
 pub struct Proofs {
     pub mmr_meta: MmrMeta,
     pub headers: Vec<Header>,
+    pub headers_with_mmr: Vec<(MmrMeta, Header)>,
     pub accounts: Vec<Account>,
     pub storages: Vec<Storage>,
     pub transactions: Vec<Transaction>,

crates/hdp_hint_processor/src/hint_processor/models/proofs/mpt.rs

@@ -6,11 +6,11 @@ use serde_with::serde_as;
 #[serde_as]
 pub struct MPTProof {
     pub block_number: u64,
-    pub proof: Bytes,
+    pub proof: Vec<Bytes>,
 }
 
 impl MPTProof {
-    pub fn new(block_number: u64, proof: Bytes) -> Self {
+    pub fn new(block_number: u64, proof: Vec<Bytes>) -> Self {
         Self { block_number, proof }
     }
 }

crates/hdp_hint_processor/src/hints/lib/verifiers/evm/account_verifier.rs

@@ -194,7 +194,11 @@ pub fn hint_get_mpt_proof(
 ) -> Result<(), HintError> {
     let proof = exec_scopes.get::<MPTProof>(vars::scopes::PROOF)?;
     let mpt_proof_ptr = get_ptr_from_var_name(vars::ids::MPT_PROOF, vm, &hint_data.ids_data, &hint_data.ap_tracking)?;
-    let proof_le_chunks: Vec<Felt252> = proof.proof.chunks(8).map(Felt252::from_bytes_le_slice).collect();
+    let proof_le_chunks: Vec<Vec<Felt252>> = proof
+        .proof
+        .into_iter()
+        .map(|p| p.chunks(8).map(Felt252::from_bytes_le_slice).collect())
+        .collect();
 
     vm.write_arg(mpt_proof_ptr, &proof_le_chunks)?;
 

crates/hdp_hint_processor/src/hints/lib/verifiers/evm/receipt_verifier.rs

@@ -153,7 +153,12 @@ pub fn hint_receipt_mpt_proof(
 ) -> Result<(), HintError> {
     let receipt = exec_scopes.get::<Receipt>(vars::scopes::RECEIPT)?;
     let mpt_proof_ptr = get_ptr_from_var_name(vars::ids::MPT_PROOF, vm, &hint_data.ids_data, &hint_data.ap_tracking)?;
-    let proof_le_chunks: Vec<Felt252> = receipt.proof.proof.chunks(8).map(Felt252::from_bytes_le_slice).collect();
+    let proof_le_chunks: Vec<Vec<Felt252>> = receipt
+        .proof
+        .proof
+        .into_iter()
+        .map(|p| p.chunks(8).map(Felt252::from_bytes_le_slice).collect())
+        .collect();
 
     vm.write_arg(mpt_proof_ptr, &proof_le_chunks)?;
 

crates/hdp_hint_processor/src/hints/lib/verifiers/evm/storage_item_verifier.rs

@@ -207,7 +207,11 @@ pub fn hint_set_mpt_proof(
 ) -> Result<(), HintError> {
     let proof = exec_scopes.get::<MPTProof>(vars::scopes::PROOF)?;
     let mpt_proof_ptr = get_ptr_from_var_name(vars::ids::MPT_PROOF, vm, &hint_data.ids_data, &hint_data.ap_tracking)?;
-    let proof_le_chunks: Vec<Felt252> = proof.proof.chunks(8).map(Felt252::from_bytes_le_slice).collect();
+    let proof_le_chunks: Vec<Vec<Felt252>> = proof
+        .proof
+        .into_iter()
+        .map(|p| p.chunks(8).map(Felt252::from_bytes_le_slice).collect())
+        .collect();
 
     vm.write_arg(mpt_proof_ptr, &proof_le_chunks)?;
 

crates/hdp_hint_processor/src/hints/lib/verifiers/evm/transaction_verifier.rs

@@ -153,7 +153,12 @@ pub fn hint_mpt_proof(
 ) -> Result<(), HintError> {
     let transaction = exec_scopes.get::<Transaction>(vars::scopes::TRANSACTION)?;
     let mpt_proof_ptr = get_ptr_from_var_name(vars::ids::MPT_PROOF, vm, &hint_data.ids_data, &hint_data.ap_tracking)?;
-    let proof_le_chunks: Vec<Felt252> = transaction.proof.proof.chunks(8).map(Felt252::from_bytes_le_slice).collect();
+    let proof_le_chunks: Vec<Vec<Felt252>> = transaction
+        .proof
+        .proof
+        .into_iter()
+        .map(|p| p.chunks(8).map(Felt252::from_bytes_le_slice).collect())
+        .collect();
 
     vm.write_arg(mpt_proof_ptr, &proof_le_chunks)?;