proof fetcher & batching refactor

crates/fetcher/src/main.rs

@@ -3,7 +3,7 @@
 use clap::{Parser, ValueHint};
 use fetcher::{proof_keys::ProofKeys, FetcherError};
 use hdp_hint_processor::{
-    hint_processor::models::proofs::{account::Account, header::Header, mmr::MmrMeta, storage::Storage, Proofs},
+    hint_processor::models::proofs::{account::Account, storage::Storage, HeaderMmrMeta, Proofs},
     syscall_handler::evm::{self, dryrun::SyscallHandler},
 };
 use std::{collections::HashSet, fs, path::PathBuf};
@@ -44,15 +44,15 @@ fn main() -> Result<(), FetcherError> {
         .header_keys
         .iter()
         .map(ProofKeys::fetch_header_proof)
-        .collect::<Result<HashSet<(MmrMeta, Header)>, FetcherError>>()?;
+        .collect::<Result<HashSet<HeaderMmrMeta>, FetcherError>>()?;
 
     let mut accounts: HashSet<Account> = HashSet::default();
 
     for (header_with_mmr, account) in proof_keys
         .account_keys
         .iter()
         .map(ProofKeys::fetch_account_proof)
-        .collect::<Result<Vec<((MmrMeta, Header), Account)>, FetcherError>>()?
+        .collect::<Result<Vec<(HeaderMmrMeta, Account)>, FetcherError>>()?
         .into_iter()
     {
         headers_with_mmr.insert(header_with_mmr);
@@ -65,7 +65,7 @@ fn main() -> Result<(), FetcherError> {
         .storage_keys
         .iter()
         .map(ProofKeys::fetch_storage_proof)
-        .collect::<Result<Vec<((MmrMeta, Header), Account, Storage)>, FetcherError>>()?
+        .collect::<Result<Vec<(HeaderMmrMeta, Account, Storage)>, FetcherError>>()?
         .into_iter()
     {
         headers_with_mmr.insert(header_with_mmr);

crates/fetcher/src/proof_keys.rs

@@ -11,6 +11,7 @@ use hdp_hint_processor::{
         mmr::MmrMeta,
         mpt::MPTProof,
         storage::Storage,
+        HeaderMmrMeta,
     },
     syscall_handler::keys,
 };
@@ -33,7 +34,12 @@ pub struct ProofKeys {
 }
 
 impl ProofKeys {
-    pub fn fetch_header_proof(key: &keys::header::Key) -> Result<(MmrMeta, Header), FetcherError> {
+    fn normalize_hex(input: &str) -> String {
+        let hex_str = input.trim_start_matches("0x");
+        format!("{:0>width$}", hex_str, width = (hex_str.len() + 1) / 2 * 2)
+    }
+
+    pub fn fetch_header_proof(key: &keys::header::Key) -> Result<HeaderMmrMeta, FetcherError> {
         let provider = Indexer::default();
         let runtime = tokio::runtime::Runtime::new().unwrap();
 
@@ -48,45 +54,54 @@ impl ProofKeys {
         let mmr_meta = MmrMeta {
             id: u64::from_str_radix(&response.mmr_meta.mmr_id[2..], 16)?,
             size: response.mmr_meta.mmr_size,
-            root: response.mmr_meta.mmr_root.parse()?,
-            chain_id: key.chain_id,
+            root: Self::normalize_hex(&response.mmr_meta.mmr_root).parse()?,
             peaks: response
                 .mmr_meta
                 .mmr_peaks
                 .iter()
-                .map(|peak| peak.parse())
+                .map(|peak| Self::normalize_hex(peak).parse())
                 .collect::<Result<Vec<Bytes>, FromHexError>>()?,
         };
 
-        // Retrieve MMR proof
         let mmr_proof = response
             .headers
             .get(&key.block_number)
             .ok_or_else(|| FetcherError::InternalError("block not found".into()))?;
 
         // Parse RLP
-        let rlp = match &mmr_proof.block_header {
-            BlockHeader::RlpString(rlp) => rlp,
+        let rlp: Bytes = match &mmr_proof.block_header {
+            BlockHeader::RlpString(rlp) => rlp.parse()?,
+            BlockHeader::RlpLittleEndian8ByteChunks(rlp) => {
+                let rlp_chunks: Vec<Bytes> = rlp
+                    .clone()
+                    .iter()
+                    .map(|x| Self::normalize_hex(x).parse())
+                    .collect::<Result<Vec<Bytes>, FromHexError>>()?;
+                rlp_chunks.iter().flat_map(|x| x.iter().rev().cloned()).collect::<Vec<u8>>().into()
+            }
             _ => return Err(FetcherError::InternalError("wrong rlp format".into())),
         };
 
         // Construct Header
         let header = Header {
-            rlp: rlp.parse()?,
+            rlp,
             proof: HeaderProof {
                 leaf_idx: mmr_proof.element_index,
                 mmr_path: mmr_proof
                     .siblings_hashes
                     .iter()
-                    .map(|hash| hash.parse())
+                    .map(|hash| Self::normalize_hex(hash).parse())
                     .collect::<Result<Vec<Bytes>, FromHexError>>()?,
             },
         };
 
-        Ok((mmr_meta, header))
+        Ok(HeaderMmrMeta {
+            mmr_meta,
+            headers: vec![header],
+        })
     }
 
-    pub fn fetch_account_proof(key: &keys::account::Key) -> Result<((MmrMeta, Header), Account), FetcherError> {
+    pub fn fetch_account_proof(key: &keys::account::Key) -> Result<(HeaderMmrMeta, Account), FetcherError> {
         let runtime = tokio::runtime::Runtime::new().unwrap();
         let provider = RootProvider::<Http<Client>>::new_http(Url::parse(&env::var(RPC).unwrap()).unwrap());
         let value = runtime
@@ -98,7 +113,7 @@ impl ProofKeys {
         ))
     }
 
-    pub fn fetch_storage_proof(key: &keys::storage::Key) -> Result<((MmrMeta, Header), Account, Storage), FetcherError> {
+    pub fn fetch_storage_proof(key: &keys::storage::Key) -> Result<(HeaderMmrMeta, Account, Storage), FetcherError> {
         let runtime = tokio::runtime::Runtime::new().unwrap();
         let provider = RootProvider::<Http<Client>>::new_http(Url::parse(&env::var(RPC).unwrap()).unwrap());
         let value = runtime

crates/hdp_hint_processor/src/hint_processor/mod.rs

@@ -103,7 +103,6 @@ impl CustomHintProcessor {
         hints.insert(lib::verifiers::evm::transaction_verifier::HINT_SET_TX_BLOCK_NUMBER.into(), lib::verifiers::evm::transaction_verifier::hint_set_tx_block_number);
         hints.insert(lib::verifiers::evm::transaction_verifier::HINT_PROOF_BYTES_LEN.into(), lib::verifiers::evm::transaction_verifier::hint_proof_bytes_len);
         hints.insert(lib::verifiers::evm::transaction_verifier::HINT_MPT_PROOF.into(), lib::verifiers::evm::transaction_verifier::hint_mpt_proof);
-        hints.insert(lib::verifiers::evm::header_verifier::HINT_BATCH_HEADERS_LEN.into(), lib::verifiers::evm::header_verifier::hint_batch_headers_len);
         hints.insert(lib::verifiers::evm::header_verifier::HINT_LEAF_IDX.into(), lib::verifiers::evm::header_verifier::hint_leaf_idx);
         hints.insert(lib::verifiers::evm::header_verifier::HINT_MMR_PATH_LEN.into(), lib::verifiers::evm::header_verifier::hint_mmr_path_len);
         hints.insert(lib::verifiers::evm::header_verifier::HINT_MMR_PATH.into(), lib::verifiers::evm::header_verifier::hint_mmr_path);
@@ -128,8 +127,6 @@ impl CustomHintProcessor {
         hints.insert(lib::verifiers::evm::storage_item_verifier::HINT_SET_STORAGE_PROOFS_LEN.into(), lib::verifiers::evm::storage_item_verifier::hint_set_storage_proofs_len);
         hints.insert(lib::verifiers::evm::storage_item_verifier::HINT_SET_STORAGE_PROOF_AT.into(), lib::verifiers::evm::storage_item_verifier::hint_set_storage_proof_at);
         hints.insert(lib::verifiers::evm::storage_item_verifier::HINT_SET_STORAGE_SLOT.into(), lib::verifiers::evm::storage_item_verifier::hint_set_storage_slot);
-        hints.insert(lib::verifiers::verify::HINT_BATCH_LEN.into(), lib::verifiers::verify::hint_batch_len);
-        hints.insert(lib::verifiers::verify::HINT_CHAIN_ID.into(), lib::verifiers::verify::hint_chain_id);
         hints.insert(lib::verifiers::verify::HINT_VM_ENTER_SCOPE.into(), lib::verifiers::verify::hint_vm_enter_scope);
         hints.insert(lib::verifiers::utils::HINT_PRINT_TASK_RESULT.into(), lib::verifiers::utils::hint_print_task_result);
         hints

crates/hdp_hint_processor/src/hint_processor/models/proofs/header.rs

@@ -2,14 +2,14 @@ use alloy::primitives::Bytes;
 use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
 
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Eq, Hash)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Eq, Hash, Default)]
 #[serde_as]
 pub struct HeaderProof {
     pub leaf_idx: u64,
     pub mmr_path: Vec<Bytes>,
 }
 
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Eq, Hash)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Eq, Hash, Default)]
 pub struct Header {
     pub rlp: Bytes,
     pub proof: HeaderProof,

crates/hdp_hint_processor/src/hint_processor/models/proofs/mmr.rs

@@ -8,7 +8,6 @@ pub struct MmrMeta {
     pub id: u64,
     pub size: u64,
     pub root: Bytes,
-    pub chain_id: u64,
     pub peaks: Vec<Bytes>,
 }
 

crates/hdp_hint_processor/src/hint_processor/models/proofs/mod.rs

@@ -14,11 +14,15 @@ use serde::{Deserialize, Serialize};
 use storage::Storage;
 use transaction::Transaction;
 
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Default, Hash)]
+pub struct HeaderMmrMeta {
+    pub headers: Vec<Header>,
+    pub mmr_meta: MmrMeta,
+}
+
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Default)]
 pub struct Proofs {
-    pub mmr_meta: MmrMeta,
-    pub headers: Vec<Header>,
-    pub headers_with_mmr: Vec<(MmrMeta, Header)>,
+    pub headers_with_mmr: Vec<HeaderMmrMeta>,
     pub accounts: Vec<Account>,
     pub storages: Vec<Storage>,
     pub transactions: Vec<Transaction>,

crates/hdp_hint_processor/src/hints/lib/verifiers/evm/header_verifier.rs

@@ -1,45 +1,69 @@
 use cairo_vm::{
     hint_processor::builtin_hint_processor::{
         builtin_hint_processor_definition::HintProcessorData,
+        dict_manager::DictManager,
         hint_utils::{get_integer_from_var_name, get_ptr_from_var_name, insert_value_into_ap},
     },
     types::exec_scope::ExecutionScopes,
     vm::{errors::hint_errors::HintError, vm_core::VirtualMachine},
     Felt252,
 };
-use std::collections::HashMap;
+use std::{any::Any, collections::HashMap};
 
 use crate::{
-    hint_processor::models::proofs::{header::Header, Proofs},
+    hint_processor::models::proofs::{header::Header, HeaderMmrMeta, Proofs},
     hints::vars,
 };
 
-pub const HINT_BATCH_HEADERS_LEN: &str = "memory[ap] = to_felt_or_relocatable(len(batch.headers))";
+pub const HINT_VM_ENTER_SCOPE: &str = "vm_enter_scope({'header_with_mmr': batch.header_with_mmr[ids.idx], '__dict_manager': __dict_manager})";
 
-pub fn hint_batch_headers_len(
+pub fn hint_vm_enter_scope(
+    vm: &mut VirtualMachine,
+    exec_scopes: &mut ExecutionScopes,
+    hint_data: &HintProcessorData,
+    _constants: &HashMap<String, Felt252>,
+) -> Result<(), HintError> {
+    let proofs = exec_scopes.get::<Proofs>(vars::scopes::BATCH)?;
+    let idx: usize = get_integer_from_var_name(vars::ids::IDX, vm, &hint_data.ids_data, &hint_data.ap_tracking)?
+        .try_into()
+        .unwrap();
+
+    let headers_with_mmr: Box<dyn Any> = Box::new(proofs.headers_with_mmr[idx].clone());
+    let dict_manager: Box<dyn Any> = Box::new(exec_scopes.get::<DictManager>(vars::scopes::DICT_MANAGER)?);
+    exec_scopes.enter_scope(HashMap::from([
+        (String::from(vars::scopes::HEADER_WITH_MMR), headers_with_mmr),
+        (String::from(vars::scopes::DICT_MANAGER), dict_manager),
+    ]));
+
+    Ok(())
+}
+
+pub const HINT_HEADERS_WITH_MMR_HEADERS_LEN: &str = "memory[ap] = to_felt_or_relocatable(len(headers_with_mmr.headers))";
+
+pub fn hint_headers_with_mmr_headers_len(
     vm: &mut VirtualMachine,
     exec_scopes: &mut ExecutionScopes,
     _hint_data: &HintProcessorData,
     _constants: &HashMap<String, Felt252>,
 ) -> Result<(), HintError> {
-    let batch = exec_scopes.get::<Proofs>(vars::scopes::BATCH)?;
+    let header_with_mmr = exec_scopes.get::<HeaderMmrMeta>(vars::scopes::HEADER_WITH_MMR)?;
 
-    insert_value_into_ap(vm, Felt252::from(batch.headers.len()))
+    insert_value_into_ap(vm, Felt252::from(header_with_mmr.headers.len()))
 }
 
-pub const HINT_SET_HEADER: &str = "header = batch.headers[ids.idx - 1]\nsegments.write_arg(ids.rlp, [int(x, 16) for x in header.rlp])";
+pub const HINT_SET_HEADER: &str = "header = header_with_mmr.headers[ids.idx - 1]\nsegments.write_arg(ids.rlp, [int(x, 16) for x in header.rlp])";
 
 pub fn hint_set_header(
     vm: &mut VirtualMachine,
     exec_scopes: &mut ExecutionScopes,
     hint_data: &HintProcessorData,
     _constants: &HashMap<String, Felt252>,
 ) -> Result<(), HintError> {
-    let batch = exec_scopes.get::<Proofs>(vars::scopes::BATCH)?;
+    let headers_with_mmr = exec_scopes.get::<HeaderMmrMeta>(vars::scopes::HEADER_WITH_MMR)?;
     let idx: usize = get_integer_from_var_name(vars::ids::IDX, vm, &hint_data.ids_data, &hint_data.ap_tracking)?
         .try_into()
         .unwrap();
-    let header = batch.headers[idx - 1].clone();
+    let header = headers_with_mmr.headers[idx - 1].clone();
     let rlp_le_chunks: Vec<Felt252> = header.rlp.chunks(8).map(Felt252::from_bytes_le_slice).collect();
 
     exec_scopes.insert_value::<Header>(vars::scopes::HEADER, header);

crates/hdp_hint_processor/src/hints/lib/verifiers/verify.rs

@@ -1,69 +1,24 @@
-use cairo_vm::{
-    hint_processor::builtin_hint_processor::{
-        builtin_hint_processor_definition::HintProcessorData, dict_manager::DictManager, hint_utils::insert_value_from_var_name,
-    },
-    types::relocatable::MaybeRelocatable,
-};
+use crate::{hint_processor::models::proofs::Proofs, hints::vars};
+use cairo_vm::hint_processor::builtin_hint_processor::hint_utils::insert_value_into_ap;
+use cairo_vm::hint_processor::builtin_hint_processor::{builtin_hint_processor_definition::HintProcessorData, dict_manager::DictManager};
 use cairo_vm::{
     types::exec_scope::ExecutionScopes,
     vm::{errors::hint_errors::HintError, vm_core::VirtualMachine},
     Felt252,
 };
 use std::{any::Any, collections::HashMap};
 
-use crate::{hint_processor::models::proofs::Proofs, hints::vars};
-
-pub const HINT_BATCH_LEN: &str = "ids.batch_len = len(proofs)";
-
-pub fn hint_batch_len(
-    vm: &mut VirtualMachine,
-    exec_scopes: &mut ExecutionScopes,
-    hint_data: &HintProcessorData,
-    _constants: &HashMap<String, Felt252>,
-) -> Result<(), HintError> {
-    let proofs = exec_scopes.get::<Vec<Proofs>>(vars::scopes::PROOFS)?;
-
-    insert_value_from_var_name(
-        vars::ids::BATCH_LEN,
-        MaybeRelocatable::Int(proofs.len().into()),
-        vm,
-        &hint_data.ids_data,
-        &hint_data.ap_tracking,
-    )
-}
-
-pub const HINT_CHAIN_ID: &str = "ids.chain_id = proofs[ids.batch_len - 1].mmr_meta.chain_id";
-
-pub fn hint_chain_id(
-    vm: &mut VirtualMachine,
-    exec_scopes: &mut ExecutionScopes,
-    hint_data: &HintProcessorData,
-    _constants: &HashMap<String, Felt252>,
-) -> Result<(), HintError> {
-    let proofs = exec_scopes.get::<Vec<Proofs>>(vars::scopes::PROOFS)?;
-
-    let chain_id = proofs[proofs.len() - 1].mmr_meta.chain_id;
-
-    insert_value_from_var_name(
-        vars::ids::CHAIN_ID,
-        MaybeRelocatable::Int(chain_id.into()),
-        vm,
-        &hint_data.ids_data,
-        &hint_data.ap_tracking,
-    )
-}
-
-pub const HINT_VM_ENTER_SCOPE: &str = "vm_enter_scope({'batch': proofs[ids.batch_len - 1], '__dict_manager': __dict_manager})";
+pub const HINT_VM_ENTER_SCOPE: &str = "vm_enter_scope({'batch': proofs, '__dict_manager': __dict_manager})";
 
 pub fn hint_vm_enter_scope(
     _vm: &mut VirtualMachine,
     exec_scopes: &mut ExecutionScopes,
     _hint_data: &HintProcessorData,
     _constants: &HashMap<String, Felt252>,
 ) -> Result<(), HintError> {
-    let proofs = exec_scopes.get::<Vec<Proofs>>(vars::scopes::PROOFS)?;
+    let proofs = exec_scopes.get::<Proofs>(vars::scopes::PROOFS)?;
 
-    let batch: Box<dyn Any> = Box::new(proofs[proofs.len() - 1].clone());
+    let batch: Box<dyn Any> = Box::new(proofs);
     let dict_manager: Box<dyn Any> = Box::new(exec_scopes.get::<DictManager>(vars::scopes::DICT_MANAGER)?);
     exec_scopes.enter_scope(HashMap::from([
         (String::from(vars::scopes::BATCH), batch),
@@ -72,3 +27,16 @@ pub fn hint_vm_enter_scope(
 
     Ok(())
 }
+
+pub const HINT_HEADERS_WITH_MMR_LEN: &str = "memory[ap] = to_felt_or_relocatable(len(batch.headers_with_mmr))";
+
+pub fn hint_headers_with_mmr_headers_len(
+    vm: &mut VirtualMachine,
+    exec_scopes: &mut ExecutionScopes,
+    _hint_data: &HintProcessorData,
+    _constants: &HashMap<String, Felt252>,
+) -> Result<(), HintError> {
+    let proofs = exec_scopes.get::<Proofs>(vars::scopes::PROOF)?;
+
+    insert_value_into_ap(vm, Felt252::from(proofs.headers_with_mmr.len()))
+}

crates/hdp_hint_processor/src/hints/vars.rs

@@ -12,6 +12,7 @@ pub mod scopes {
     pub const STORAGE: &str = "storage";
     pub const SYSCALL_HANDLER: &str = "syscall_handler";
     pub const TRANSACTION: &str = "transaction";
+    pub const HEADER_WITH_MMR: &str = "header_with_mmr";
 }
 
 pub mod ids {

crates/provider/src/indexer/types.rs

@@ -59,6 +59,7 @@ pub struct IndexerQuery {
     pub is_whole_tree: bool,
     pub is_rlp_included: bool,
     pub is_pure_rlp: bool,
+    pub prefer_native_block_header: bool,
 }
 
 impl IndexerQuery {
@@ -74,6 +75,7 @@ impl IndexerQuery {
             is_whole_tree: true,
             is_rlp_included: true,
             is_pure_rlp: true,
+            prefer_native_block_header: false,
         }
     }
 }

src/types.cairo

@@ -11,7 +11,6 @@ struct MMRMeta {
     id: felt,
     root: felt,
     size: felt,
-    chain_id: felt,
 }
 
 struct ModuleTask {